@hashgraphonline/standards-sdk 0.1.190 → 0.1.191
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/hcs-25/adapters/ans-trust-discovery.d.ts +22 -0
- package/dist/browser/hcs-25/adapters/ans-trust-discovery.d.ts.map +1 -0
- package/dist/browser/hcs-25/adapters/index.d.ts +1 -0
- package/dist/browser/hcs-25/adapters/index.d.ts.map +1 -1
- package/dist/browser/hcs-25/collectors/ans-trust-discovery.d.ts +32 -0
- package/dist/browser/hcs-25/collectors/ans-trust-discovery.d.ts.map +1 -0
- package/dist/browser/hcs-25/collectors/catalog.d.ts +9 -2
- package/dist/browser/hcs-25/collectors/catalog.d.ts.map +1 -1
- package/dist/browser/hcs-25/collectors/index.d.ts +1 -0
- package/dist/browser/hcs-25/collectors/index.d.ts.map +1 -1
- package/dist/browser/hcs-25/signals/ans-trust-discovery.d.ts +50 -0
- package/dist/browser/hcs-25/signals/ans-trust-discovery.d.ts.map +1 -0
- package/dist/browser/hcs-25/signals/index.d.ts +1 -0
- package/dist/browser/hcs-25/signals/index.d.ts.map +1 -1
- package/dist/browser-root/hcs-25/adapters/ans-trust-discovery.d.ts +22 -0
- package/dist/browser-root/hcs-25/adapters/ans-trust-discovery.d.ts.map +1 -0
- package/dist/browser-root/hcs-25/adapters/index.d.ts +1 -0
- package/dist/browser-root/hcs-25/adapters/index.d.ts.map +1 -1
- package/dist/browser-root/hcs-25/collectors/ans-trust-discovery.d.ts +32 -0
- package/dist/browser-root/hcs-25/collectors/ans-trust-discovery.d.ts.map +1 -0
- package/dist/browser-root/hcs-25/collectors/catalog.d.ts +9 -2
- package/dist/browser-root/hcs-25/collectors/catalog.d.ts.map +1 -1
- package/dist/browser-root/hcs-25/collectors/index.d.ts +1 -0
- package/dist/browser-root/hcs-25/collectors/index.d.ts.map +1 -1
- package/dist/browser-root/hcs-25/signals/ans-trust-discovery.d.ts +50 -0
- package/dist/browser-root/hcs-25/signals/ans-trust-discovery.d.ts.map +1 -0
- package/dist/browser-root/hcs-25/signals/index.d.ts +1 -0
- package/dist/browser-root/hcs-25/signals/index.d.ts.map +1 -1
- package/dist/cjs/hcs-25/adapters/ans-trust-discovery.d.ts +22 -0
- package/dist/cjs/hcs-25/adapters/ans-trust-discovery.d.ts.map +1 -0
- package/dist/cjs/hcs-25/adapters/index.d.ts +1 -0
- package/dist/cjs/hcs-25/adapters/index.d.ts.map +1 -1
- package/dist/cjs/hcs-25/collectors/ans-trust-discovery.d.ts +32 -0
- package/dist/cjs/hcs-25/collectors/ans-trust-discovery.d.ts.map +1 -0
- package/dist/cjs/hcs-25/collectors/catalog.d.ts +9 -2
- package/dist/cjs/hcs-25/collectors/catalog.d.ts.map +1 -1
- package/dist/cjs/hcs-25/collectors/index.d.ts +1 -0
- package/dist/cjs/hcs-25/collectors/index.d.ts.map +1 -1
- package/dist/cjs/hcs-25/signals/ans-trust-discovery.d.ts +50 -0
- package/dist/cjs/hcs-25/signals/ans-trust-discovery.d.ts.map +1 -0
- package/dist/cjs/hcs-25/signals/index.d.ts +1 -0
- package/dist/cjs/hcs-25/signals/index.d.ts.map +1 -1
- package/dist/cjs/standards-sdk.cjs +1 -1
- package/dist/cjs/standards-sdk.cjs.map +1 -1
- package/dist/es/hcs-25/adapters/ans-trust-discovery.d.ts +22 -0
- package/dist/es/hcs-25/adapters/ans-trust-discovery.d.ts.map +1 -0
- package/dist/es/hcs-25/adapters/index.d.ts +1 -0
- package/dist/es/hcs-25/adapters/index.d.ts.map +1 -1
- package/dist/es/hcs-25/collectors/ans-trust-discovery.d.ts +32 -0
- package/dist/es/hcs-25/collectors/ans-trust-discovery.d.ts.map +1 -0
- package/dist/es/hcs-25/collectors/catalog.d.ts +9 -2
- package/dist/es/hcs-25/collectors/catalog.d.ts.map +1 -1
- package/dist/es/hcs-25/collectors/index.d.ts +1 -0
- package/dist/es/hcs-25/collectors/index.d.ts.map +1 -1
- package/dist/es/hcs-25/signals/ans-trust-discovery.d.ts +50 -0
- package/dist/es/hcs-25/signals/ans-trust-discovery.d.ts.map +1 -0
- package/dist/es/hcs-25/signals/index.d.ts +1 -0
- package/dist/es/hcs-25/signals/index.d.ts.map +1 -1
- package/dist/es/standards-sdk.es.js +78 -71
- package/dist/es/standards-sdk.es.js.map +1 -1
- package/dist/es/standards-sdk.es101.js +2 -2
- package/dist/es/standards-sdk.es103.js +2 -2
- package/dist/es/standards-sdk.es104.js +1 -1
- package/dist/es/standards-sdk.es106.js +1 -1
- package/dist/es/standards-sdk.es108.js +2 -2
- package/dist/es/standards-sdk.es11.js +1 -1
- package/dist/es/standards-sdk.es110.js +1 -1
- package/dist/es/standards-sdk.es111.js +2 -2
- package/dist/es/standards-sdk.es115.js +3 -3
- package/dist/es/standards-sdk.es116.js +4 -4
- package/dist/es/standards-sdk.es12.js +1 -1
- package/dist/es/standards-sdk.es120.js +22 -4
- package/dist/es/standards-sdk.es120.js.map +1 -1
- package/dist/es/standards-sdk.es121.js +15 -12
- package/dist/es/standards-sdk.es121.js.map +1 -1
- package/dist/es/standards-sdk.es123.js +6 -5
- package/dist/es/standards-sdk.es123.js.map +1 -1
- package/dist/es/standards-sdk.es124.js +6 -5
- package/dist/es/standards-sdk.es124.js.map +1 -1
- package/dist/es/standards-sdk.es127.js +6 -5
- package/dist/es/standards-sdk.es127.js.map +1 -1
- package/dist/es/standards-sdk.es128.js +110 -136
- package/dist/es/standards-sdk.es128.js.map +1 -1
- package/dist/es/standards-sdk.es129.js +133 -141
- package/dist/es/standards-sdk.es129.js.map +1 -1
- package/dist/es/standards-sdk.es13.js +1 -1
- package/dist/es/standards-sdk.es130.js +135 -345
- package/dist/es/standards-sdk.es130.js.map +1 -1
- package/dist/es/standards-sdk.es131.js +332 -344
- package/dist/es/standards-sdk.es131.js.map +1 -1
- package/dist/es/standards-sdk.es132.js +338 -178
- package/dist/es/standards-sdk.es132.js.map +1 -1
- package/dist/es/standards-sdk.es133.js +200 -196
- package/dist/es/standards-sdk.es133.js.map +1 -1
- package/dist/es/standards-sdk.es134.js +216 -364
- package/dist/es/standards-sdk.es134.js.map +1 -1
- package/dist/es/standards-sdk.es135.js +358 -100
- package/dist/es/standards-sdk.es135.js.map +1 -1
- package/dist/es/standards-sdk.es136.js +91 -220
- package/dist/es/standards-sdk.es136.js.map +1 -1
- package/dist/es/standards-sdk.es137.js +173 -1250
- package/dist/es/standards-sdk.es137.js.map +1 -1
- package/dist/es/standards-sdk.es138.js +1295 -98
- package/dist/es/standards-sdk.es138.js.map +1 -1
- package/dist/es/standards-sdk.es139.js +109 -246
- package/dist/es/standards-sdk.es139.js.map +1 -1
- package/dist/es/standards-sdk.es14.js +1 -1
- package/dist/es/standards-sdk.es140.js +263 -97
- package/dist/es/standards-sdk.es140.js.map +1 -1
- package/dist/es/standards-sdk.es141.js +107 -248
- package/dist/es/standards-sdk.es141.js.map +1 -1
- package/dist/es/standards-sdk.es142.js +246 -472
- package/dist/es/standards-sdk.es142.js.map +1 -1
- package/dist/es/standards-sdk.es143.js +458 -85
- package/dist/es/standards-sdk.es143.js.map +1 -1
- package/dist/es/standards-sdk.es144.js +96 -151
- package/dist/es/standards-sdk.es144.js.map +1 -1
- package/dist/es/standards-sdk.es145.js +155 -29
- package/dist/es/standards-sdk.es145.js.map +1 -1
- package/dist/es/standards-sdk.es146.js +26 -4
- package/dist/es/standards-sdk.es146.js.map +1 -1
- package/dist/es/standards-sdk.es147.js +10 -180
- package/dist/es/standards-sdk.es147.js.map +1 -1
- package/dist/es/standards-sdk.es148.js +179 -26
- package/dist/es/standards-sdk.es148.js.map +1 -1
- package/dist/es/standards-sdk.es149.js +27 -20
- package/dist/es/standards-sdk.es149.js.map +1 -1
- package/dist/es/standards-sdk.es15.js +1 -1
- package/dist/es/standards-sdk.es150.js +18 -156
- package/dist/es/standards-sdk.es150.js.map +1 -1
- package/dist/es/standards-sdk.es151.js +148 -190
- package/dist/es/standards-sdk.es151.js.map +1 -1
- package/dist/es/standards-sdk.es152.js +159 -747
- package/dist/es/standards-sdk.es152.js.map +1 -1
- package/dist/es/standards-sdk.es153.js +786 -9
- package/dist/es/standards-sdk.es153.js.map +1 -1
- package/dist/es/standards-sdk.es154.js +13 -567
- package/dist/es/standards-sdk.es154.js.map +1 -1
- package/dist/es/standards-sdk.es155.js +541 -600
- package/dist/es/standards-sdk.es155.js.map +1 -1
- package/dist/es/standards-sdk.es156.js +625 -12
- package/dist/es/standards-sdk.es156.js.map +1 -1
- package/dist/es/standards-sdk.es157.js +13 -2
- package/dist/es/standards-sdk.es157.js.map +1 -1
- package/dist/es/standards-sdk.es158.js +2 -87
- package/dist/es/standards-sdk.es158.js.map +1 -1
- package/dist/es/standards-sdk.es159.js +84 -37
- package/dist/es/standards-sdk.es159.js.map +1 -1
- package/dist/es/standards-sdk.es16.js +6 -6
- package/dist/es/standards-sdk.es160.js +40 -2
- package/dist/es/standards-sdk.es160.js.map +1 -1
- package/dist/es/standards-sdk.es161.js +2 -235
- package/dist/es/standards-sdk.es161.js.map +1 -1
- package/dist/es/standards-sdk.es162.js +204 -1109
- package/dist/es/standards-sdk.es162.js.map +1 -1
- package/dist/es/standards-sdk.es163.js +1059 -225
- package/dist/es/standards-sdk.es163.js.map +1 -1
- package/dist/es/standards-sdk.es164.js +303 -419
- package/dist/es/standards-sdk.es164.js.map +1 -1
- package/dist/es/standards-sdk.es165.js +418 -351
- package/dist/es/standards-sdk.es165.js.map +1 -1
- package/dist/es/standards-sdk.es166.js +347 -1114
- package/dist/es/standards-sdk.es166.js.map +1 -1
- package/dist/es/standards-sdk.es167.js +1094 -180
- package/dist/es/standards-sdk.es167.js.map +1 -1
- package/dist/es/standards-sdk.es168.js +153 -1512
- package/dist/es/standards-sdk.es168.js.map +1 -1
- package/dist/es/standards-sdk.es169.js +1457 -1056
- package/dist/es/standards-sdk.es169.js.map +1 -1
- package/dist/es/standards-sdk.es170.js +1163 -14
- package/dist/es/standards-sdk.es170.js.map +1 -1
- package/dist/es/standards-sdk.es171.js +15 -449
- package/dist/es/standards-sdk.es171.js.map +1 -1
- package/dist/es/standards-sdk.es172.js +440 -248
- package/dist/es/standards-sdk.es172.js.map +1 -1
- package/dist/es/standards-sdk.es173.js +259 -4
- package/dist/es/standards-sdk.es173.js.map +1 -1
- package/dist/es/standards-sdk.es174.js +4 -81
- package/dist/es/standards-sdk.es174.js.map +1 -1
- package/dist/es/standards-sdk.es175.js +71 -77
- package/dist/es/standards-sdk.es175.js.map +1 -1
- package/dist/es/standards-sdk.es176.js +80 -53
- package/dist/es/standards-sdk.es176.js.map +1 -1
- package/dist/es/standards-sdk.es177.js +59 -2546
- package/dist/es/standards-sdk.es177.js.map +1 -1
- package/dist/es/standards-sdk.es178.js +2547 -159
- package/dist/es/standards-sdk.es178.js.map +1 -1
- package/dist/es/standards-sdk.es179.js +159 -7
- package/dist/es/standards-sdk.es179.js.map +1 -1
- package/dist/es/standards-sdk.es18.js +13 -13
- package/dist/es/standards-sdk.es180.js +7 -86
- package/dist/es/standards-sdk.es180.js.map +1 -1
- package/dist/es/standards-sdk.es181.js +65 -44
- package/dist/es/standards-sdk.es181.js.map +1 -1
- package/dist/es/standards-sdk.es182.js +57 -188
- package/dist/es/standards-sdk.es182.js.map +1 -1
- package/dist/es/standards-sdk.es183.js +27 -26
- package/dist/es/standards-sdk.es183.js.map +1 -1
- package/dist/es/standards-sdk.es184.js +191 -53
- package/dist/es/standards-sdk.es184.js.map +1 -1
- package/dist/es/standards-sdk.es185.js +21 -109
- package/dist/es/standards-sdk.es185.js.map +1 -1
- package/dist/es/standards-sdk.es186.js +49 -47
- package/dist/es/standards-sdk.es186.js.map +1 -1
- package/dist/es/standards-sdk.es187.js +89 -205
- package/dist/es/standards-sdk.es187.js.map +1 -1
- package/dist/es/standards-sdk.es188.js +35 -58
- package/dist/es/standards-sdk.es188.js.map +1 -1
- package/dist/es/standards-sdk.es189.js +48 -39
- package/dist/es/standards-sdk.es189.js.map +1 -1
- package/dist/es/standards-sdk.es19.js +10 -10
- package/dist/es/standards-sdk.es190.js +218 -48
- package/dist/es/standards-sdk.es190.js.map +1 -1
- package/dist/es/standards-sdk.es191.js +66 -42
- package/dist/es/standards-sdk.es191.js.map +1 -1
- package/dist/es/standards-sdk.es192.js +33 -105
- package/dist/es/standards-sdk.es192.js.map +1 -1
- package/dist/es/standards-sdk.es193.js +49 -43
- package/dist/es/standards-sdk.es193.js.map +1 -1
- package/dist/es/standards-sdk.es194.js +33 -42
- package/dist/es/standards-sdk.es194.js.map +1 -1
- package/dist/es/standards-sdk.es195.js +123 -72
- package/dist/es/standards-sdk.es195.js.map +1 -1
- package/dist/es/standards-sdk.es196.js +58 -75
- package/dist/es/standards-sdk.es196.js.map +1 -1
- package/dist/es/standards-sdk.es197.js +60 -44
- package/dist/es/standards-sdk.es197.js.map +1 -1
- package/dist/es/standards-sdk.es198.js +72 -36
- package/dist/es/standards-sdk.es198.js.map +1 -1
- package/dist/es/standards-sdk.es199.js +75 -30
- package/dist/es/standards-sdk.es199.js.map +1 -1
- package/dist/es/standards-sdk.es2.js +2 -2
- package/dist/es/standards-sdk.es20.js +1 -1
- package/dist/es/standards-sdk.es200.js +44 -34
- package/dist/es/standards-sdk.es200.js.map +1 -1
- package/dist/es/standards-sdk.es201.js +36 -48
- package/dist/es/standards-sdk.es201.js.map +1 -1
- package/dist/es/standards-sdk.es202.js +30 -138
- package/dist/es/standards-sdk.es202.js.map +1 -1
- package/dist/es/standards-sdk.es203.js +34 -42
- package/dist/es/standards-sdk.es203.js.map +1 -1
- package/dist/es/standards-sdk.es204.js +48 -12482
- package/dist/es/standards-sdk.es204.js.map +1 -1
- package/dist/es/standards-sdk.es205.js +134 -13
- package/dist/es/standards-sdk.es205.js.map +1 -1
- package/dist/es/standards-sdk.es206.js +36 -50
- package/dist/es/standards-sdk.es206.js.map +1 -1
- package/dist/es/standards-sdk.es207.js +12449 -78
- package/dist/es/standards-sdk.es207.js.map +1 -1
- package/dist/es/standards-sdk.es208.js +15 -79
- package/dist/es/standards-sdk.es208.js.map +1 -1
- package/dist/es/standards-sdk.es209.js +48 -192
- package/dist/es/standards-sdk.es209.js.map +1 -1
- package/dist/es/standards-sdk.es21.js +1 -1
- package/dist/es/standards-sdk.es210.js +90 -651
- package/dist/es/standards-sdk.es210.js.map +1 -1
- package/dist/es/standards-sdk.es211.js +71 -63
- package/dist/es/standards-sdk.es211.js.map +1 -1
- package/dist/es/standards-sdk.es212.js +188 -160
- package/dist/es/standards-sdk.es212.js.map +1 -1
- package/dist/es/standards-sdk.es213.js +65 -314
- package/dist/es/standards-sdk.es213.js.map +1 -1
- package/dist/es/standards-sdk.es214.js +661 -335
- package/dist/es/standards-sdk.es214.js.map +1 -1
- package/dist/es/standards-sdk.es215.js +129 -410
- package/dist/es/standards-sdk.es215.js.map +1 -1
- package/dist/es/standards-sdk.es216.js +200 -209
- package/dist/es/standards-sdk.es216.js.map +1 -1
- package/dist/es/standards-sdk.es217.js +334 -64
- package/dist/es/standards-sdk.es217.js.map +1 -1
- package/dist/es/standards-sdk.es218.js +438 -124
- package/dist/es/standards-sdk.es218.js.map +1 -1
- package/dist/es/standards-sdk.es219.js +310 -496
- package/dist/es/standards-sdk.es219.js.map +1 -1
- package/dist/es/standards-sdk.es22.js +1 -1
- package/dist/es/standards-sdk.es220.js +60 -243
- package/dist/es/standards-sdk.es220.js.map +1 -1
- package/dist/es/standards-sdk.es221.js +118 -175
- package/dist/es/standards-sdk.es221.js.map +1 -1
- package/dist/es/standards-sdk.es222.js +465 -67
- package/dist/es/standards-sdk.es222.js.map +1 -1
- package/dist/es/standards-sdk.es223.js +250 -143
- package/dist/es/standards-sdk.es223.js.map +1 -1
- package/dist/es/standards-sdk.es224.js +172 -160
- package/dist/es/standards-sdk.es224.js.map +1 -1
- package/dist/es/standards-sdk.es225.js +65 -630
- package/dist/es/standards-sdk.es225.js.map +1 -1
- package/dist/es/standards-sdk.es226.js +128 -161
- package/dist/es/standards-sdk.es226.js.map +1 -1
- package/dist/es/standards-sdk.es227.js +144 -195
- package/dist/es/standards-sdk.es227.js.map +1 -1
- package/dist/es/standards-sdk.es228.js +668 -223
- package/dist/es/standards-sdk.es228.js.map +1 -1
- package/dist/es/standards-sdk.es229.js +188 -0
- package/dist/es/standards-sdk.es229.js.map +1 -0
- package/dist/es/standards-sdk.es23.js +1 -1
- package/dist/es/standards-sdk.es230.js +238 -0
- package/dist/es/standards-sdk.es230.js.map +1 -0
- package/dist/es/standards-sdk.es231.js +242 -0
- package/dist/es/standards-sdk.es231.js.map +1 -0
- package/dist/es/standards-sdk.es24.js +1 -1
- package/dist/es/standards-sdk.es25.js +1 -1
- package/dist/es/standards-sdk.es26.js +1 -1
- package/dist/es/standards-sdk.es27.js +13 -13
- package/dist/es/standards-sdk.es30.js +2 -2
- package/dist/es/standards-sdk.es31.js +4 -4
- package/dist/es/standards-sdk.es32.js +1 -1
- package/dist/es/standards-sdk.es35.js +7 -7
- package/dist/es/standards-sdk.es36.js +4 -4
- package/dist/es/standards-sdk.es37.js +2 -2
- package/dist/es/standards-sdk.es38.js +2 -2
- package/dist/es/standards-sdk.es39.js +1 -1
- package/dist/es/standards-sdk.es4.js +2 -2
- package/dist/es/standards-sdk.es40.js +1 -1
- package/dist/es/standards-sdk.es41.js +2 -2
- package/dist/es/standards-sdk.es46.js +1 -1
- package/dist/es/standards-sdk.es5.js +2 -2
- package/dist/es/standards-sdk.es51.js +1 -1
- package/dist/es/standards-sdk.es53.js +1 -1
- package/dist/es/standards-sdk.es56.js +2 -2
- package/dist/es/standards-sdk.es59.js +1 -1
- package/dist/es/standards-sdk.es6.js +2 -2
- package/dist/es/standards-sdk.es60.js +1 -1
- package/dist/es/standards-sdk.es62.js +1 -1
- package/dist/es/standards-sdk.es63.js +2 -2
- package/dist/es/standards-sdk.es64.js +1 -1
- package/dist/es/standards-sdk.es65.js +1 -1
- package/dist/es/standards-sdk.es66.js +1 -1
- package/dist/es/standards-sdk.es67.js +8 -8
- package/dist/es/standards-sdk.es69.js +1 -1
- package/dist/es/standards-sdk.es7.js +1 -1
- package/dist/es/standards-sdk.es71.js +2 -2
- package/dist/es/standards-sdk.es72.js +3 -3
- package/dist/es/standards-sdk.es75.js +5 -5
- package/dist/es/standards-sdk.es76.js +3 -3
- package/dist/es/standards-sdk.es77.js +2 -2
- package/dist/es/standards-sdk.es78.js +1 -1
- package/dist/es/standards-sdk.es81.js +2 -2
- package/dist/es/standards-sdk.es83.js +2 -2
- package/dist/es/standards-sdk.es84.js +5 -5
- package/dist/es/standards-sdk.es85.js +1 -1
- package/dist/es/standards-sdk.es88.js +1 -1
- package/dist/es/standards-sdk.es89.js +2 -2
- package/dist/es/standards-sdk.es9.js +2 -2
- package/dist/es/standards-sdk.es90.js +4 -4
- package/dist/es/standards-sdk.es94.js +3 -3
- package/dist/es/standards-sdk.es96.js +2 -2
- package/dist/es/standards-sdk.es98.js +1 -1
- package/dist/es/standards-sdk.es99.js +3 -3
- package/package.json +1 -1
|
@@ -1,1332 +1,255 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import "./standards-sdk.
|
|
3
|
-
import "./standards-sdk.
|
|
4
|
-
import "./standards-sdk.
|
|
5
|
-
import "./standards-sdk.
|
|
6
|
-
import
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
const DEFAULT_MODEL_REGISTRIES = ["openrouter", "near-ai"];
|
|
15
|
-
const OPENROUTER_CATEGORY_SOURCE = "openrouter:category-rankings";
|
|
16
|
-
const HUGGINGFACE_MODEL_INDEX_SOURCE = "huggingface:model-index";
|
|
17
|
-
const HUGGINGFACE_POPULARITY_SOURCE = "huggingface:popularity";
|
|
18
|
-
const CHATBOT_ARENA_SOURCE = "chatbot-arena:leaderboard";
|
|
19
|
-
const OPENLLM_EVAL_SOURCES = ["openllm-leaderboard"];
|
|
20
|
-
const MODEL_ID_KEYS = [
|
|
21
|
-
"openrouterModelId",
|
|
22
|
-
"huggingFaceModelId",
|
|
23
|
-
"openrouterHuggingFaceId",
|
|
24
|
-
"modelId",
|
|
25
|
-
"model",
|
|
26
|
-
"name"
|
|
27
|
-
];
|
|
28
|
-
function resolveModelId(subject) {
|
|
1
|
+
import { readSubjectAdditional, readString } from "./standards-sdk.es201.js";
|
|
2
|
+
import "./standards-sdk.es184.js";
|
|
3
|
+
import "./standards-sdk.es198.js";
|
|
4
|
+
import "./standards-sdk.es199.js";
|
|
5
|
+
import "./standards-sdk.es200.js";
|
|
6
|
+
import "./standards-sdk.es183.js";
|
|
7
|
+
import { parseTimestampMs } from "./standards-sdk.es123.js";
|
|
8
|
+
import { requestJson, Hcs25CollectorHttpError, isTimeoutError } from "./standards-sdk.es122.js";
|
|
9
|
+
const AGENTVERSE_API_BASE = "https://agentverse.ai";
|
|
10
|
+
const INSIGHTS_SOURCES = ["agentverse:insights:v1"];
|
|
11
|
+
const AGENTVERSE_ADDRESS_REGEX = /^agent1[qpzry9x8gf2tvdw0s3jn54khce6mua7l]{59}$/i;
|
|
12
|
+
const DEFAULT_INCLUDED_REGISTRIES = ["agentverse", "uagent"];
|
|
13
|
+
function resolveAgentAddress(subject) {
|
|
29
14
|
const additional = readSubjectAdditional(subject);
|
|
30
|
-
for (const key of MODEL_ID_KEYS) {
|
|
31
|
-
const value = readString(additional, key);
|
|
32
|
-
if (value) {
|
|
33
|
-
return value;
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
15
|
const metadata = subject.metadata ?? {};
|
|
37
|
-
for (const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
16
|
+
for (const value of [
|
|
17
|
+
readString(additional, "agentverseInsightsAddress"),
|
|
18
|
+
subject.id,
|
|
19
|
+
readString(metadata, "nativeId"),
|
|
20
|
+
readString(metadata, "uid"),
|
|
21
|
+
readString(metadata, "address")
|
|
22
|
+
]) {
|
|
23
|
+
if (value && AGENTVERSE_ADDRESS_REGEX.test(value.trim())) {
|
|
24
|
+
return value.trim();
|
|
41
25
|
}
|
|
42
26
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (a === b) {
|
|
49
|
-
return true;
|
|
50
|
-
}
|
|
51
|
-
const stripOrg = (id) => id.includes("/") ? id.slice(id.lastIndexOf("/") + 1) : id;
|
|
52
|
-
return stripOrg(a) === stripOrg(b);
|
|
53
|
-
}
|
|
54
|
-
function statusFromError(error) {
|
|
55
|
-
return isTimeoutError(error) ? "timeout" : "error";
|
|
56
|
-
}
|
|
57
|
-
function asRecord(value) {
|
|
58
|
-
return isJsonObject(value) ? value : void 0;
|
|
59
|
-
}
|
|
60
|
-
function parseNumber(value) {
|
|
61
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
62
|
-
return value;
|
|
63
|
-
}
|
|
64
|
-
if (typeof value === "string" && value.trim()) {
|
|
65
|
-
const parsed = Number(value.trim());
|
|
66
|
-
return Number.isFinite(parsed) ? parsed : null;
|
|
67
|
-
}
|
|
68
|
-
return null;
|
|
69
|
-
}
|
|
70
|
-
const encodeModelIdForPath = (modelId) => modelId.split("/").map((segment) => encodeURIComponent(segment)).join("/");
|
|
71
|
-
const METRIC_EXCLUDE_TOKENS = [
|
|
72
|
-
"loss",
|
|
73
|
-
"perplexity",
|
|
74
|
-
"ppl",
|
|
75
|
-
"error",
|
|
76
|
-
"rmse",
|
|
77
|
-
"mae",
|
|
78
|
-
"wer",
|
|
79
|
-
"cer",
|
|
80
|
-
"bits"
|
|
81
|
-
];
|
|
82
|
-
const METRIC_INCLUDE_TOKENS = [
|
|
83
|
-
"accuracy",
|
|
84
|
-
"acc",
|
|
85
|
-
"exact",
|
|
86
|
-
"em",
|
|
87
|
-
"f1",
|
|
88
|
-
"precision",
|
|
89
|
-
"recall",
|
|
90
|
-
"score",
|
|
91
|
-
"pass",
|
|
92
|
-
"win",
|
|
93
|
-
"bleu",
|
|
94
|
-
"rouge",
|
|
95
|
-
"mmlu",
|
|
96
|
-
"gsm",
|
|
97
|
-
"arc",
|
|
98
|
-
"hellaswag",
|
|
99
|
-
"truthfulqa",
|
|
100
|
-
"gpqa",
|
|
101
|
-
"humaneval",
|
|
102
|
-
"mbpp",
|
|
103
|
-
"swe",
|
|
104
|
-
"math",
|
|
105
|
-
"bbh",
|
|
106
|
-
"ceval"
|
|
107
|
-
];
|
|
108
|
-
const hasToken = (input, tokens) => tokens.some((token) => input.includes(token));
|
|
109
|
-
function shouldIncludeMetric(name) {
|
|
110
|
-
const normalized = name.trim().toLowerCase();
|
|
111
|
-
if (!normalized) {
|
|
112
|
-
return false;
|
|
113
|
-
}
|
|
114
|
-
if (hasToken(normalized, METRIC_EXCLUDE_TOKENS)) {
|
|
115
|
-
return false;
|
|
116
|
-
}
|
|
117
|
-
return hasToken(normalized, METRIC_INCLUDE_TOKENS);
|
|
118
|
-
}
|
|
119
|
-
function normalizeMetricValue(raw) {
|
|
120
|
-
if (!Number.isFinite(raw)) {
|
|
121
|
-
return null;
|
|
122
|
-
}
|
|
123
|
-
if (raw <= 1) {
|
|
124
|
-
return Math.min(100, Math.max(0, raw * 100));
|
|
125
|
-
}
|
|
126
|
-
if (raw <= 100) {
|
|
127
|
-
return Math.min(100, Math.max(0, raw));
|
|
128
|
-
}
|
|
129
|
-
return null;
|
|
130
|
-
}
|
|
131
|
-
function normalizeLogScore(value, max) {
|
|
132
|
-
if (!Number.isFinite(value) || value <= 0 || !Number.isFinite(max) || max <= 0) {
|
|
133
|
-
return null;
|
|
134
|
-
}
|
|
135
|
-
const numerator = Math.log10(value + 1);
|
|
136
|
-
const denominator = Math.log10(max + 1);
|
|
137
|
-
if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator <= 0) {
|
|
138
|
-
return null;
|
|
139
|
-
}
|
|
140
|
-
return Math.min(100, Math.max(0, numerator / denominator * 100));
|
|
141
|
-
}
|
|
142
|
-
function computePopularityScore(downloads, likes, downloadsCap = 1e7, likesCap = 1e5) {
|
|
143
|
-
const downloadsScore = downloads !== null ? normalizeLogScore(downloads, downloadsCap) : null;
|
|
144
|
-
const likesScore = likes !== null ? normalizeLogScore(likes, likesCap) : null;
|
|
145
|
-
if (downloadsScore === null) {
|
|
146
|
-
return likesScore;
|
|
147
|
-
}
|
|
148
|
-
if (likesScore === null) {
|
|
149
|
-
return downloadsScore;
|
|
150
|
-
}
|
|
151
|
-
return downloadsScore * 0.7 + likesScore * 0.3;
|
|
152
|
-
}
|
|
153
|
-
function extractModelIndexMetrics(payload, maxMetrics = 60) {
|
|
154
|
-
const modelIndex = payload?.["model-index"] ?? payload?.model_index ?? payload?.modelIndex ?? payload?.cardData?.["model-index"];
|
|
155
|
-
if (!Array.isArray(modelIndex)) {
|
|
156
|
-
return [];
|
|
157
|
-
}
|
|
158
|
-
const metrics = [];
|
|
159
|
-
for (const entry of modelIndex) {
|
|
160
|
-
const results = asRecord(entry)?.results;
|
|
161
|
-
if (!Array.isArray(results)) {
|
|
162
|
-
continue;
|
|
163
|
-
}
|
|
164
|
-
for (const result of results) {
|
|
165
|
-
const block = asRecord(result)?.metrics;
|
|
166
|
-
const push = (name, rawValue) => {
|
|
167
|
-
const rawName = typeof name === "string" && name.trim() ? name.trim() : null;
|
|
168
|
-
const parsed = parseNumber(rawValue);
|
|
169
|
-
if (!rawName || parsed === null || !shouldIncludeMetric(rawName)) {
|
|
170
|
-
return;
|
|
171
|
-
}
|
|
172
|
-
const normalized = normalizeMetricValue(parsed);
|
|
173
|
-
if (normalized === null) {
|
|
174
|
-
return;
|
|
175
|
-
}
|
|
176
|
-
metrics.push({ name: rawName, value: normalized });
|
|
177
|
-
};
|
|
178
|
-
if (Array.isArray(block)) {
|
|
179
|
-
for (const metric of block) {
|
|
180
|
-
const record = asRecord(metric);
|
|
181
|
-
if (!record) {
|
|
182
|
-
continue;
|
|
183
|
-
}
|
|
184
|
-
const name = record.name ?? record.metric ?? record.type ?? null;
|
|
185
|
-
push(name, record.value);
|
|
186
|
-
if (metrics.length >= maxMetrics) {
|
|
187
|
-
return metrics;
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
} else if (block && typeof block === "object") {
|
|
191
|
-
for (const [name, value] of Object.entries(
|
|
192
|
-
block
|
|
193
|
-
)) {
|
|
194
|
-
push(name, value);
|
|
195
|
-
if (metrics.length >= maxMetrics) {
|
|
196
|
-
return metrics;
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
}
|
|
27
|
+
const uaidFull = readString(metadata, "uaidFull");
|
|
28
|
+
if (uaidFull) {
|
|
29
|
+
const nativeId = uaidFull.match(/(?:^|;)nativeId=([^;]+)(?:;|$)/i)?.[1]?.trim();
|
|
30
|
+
if (nativeId && AGENTVERSE_ADDRESS_REGEX.test(nativeId)) {
|
|
31
|
+
return nativeId;
|
|
200
32
|
}
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
async function resolveHuggingFaceModelId(modelId, context, options = {}) {
|
|
205
|
-
const normalized = modelId.trim().toLowerCase();
|
|
206
|
-
if (!normalized) {
|
|
207
|
-
return null;
|
|
208
|
-
}
|
|
209
|
-
try {
|
|
210
|
-
const payload = await requestJson(`${options.openrouterApiBase ?? OPENROUTER_API_BASE}/models`, {
|
|
211
|
-
fetch: context.fetch,
|
|
212
|
-
timeoutMs: context.timeoutMs,
|
|
213
|
-
signal: context.signal
|
|
214
|
-
});
|
|
215
|
-
if (Array.isArray(payload.data)) {
|
|
216
|
-
const match = payload.data.find((entry) => {
|
|
217
|
-
const id = typeof entry.id === "string" ? entry.id.trim() : "";
|
|
218
|
-
return id.toLowerCase() === normalized;
|
|
219
|
-
});
|
|
220
|
-
const hfId = match && typeof match.hugging_face_id === "string" ? match.hugging_face_id.trim() : "";
|
|
221
|
-
if (hfId) {
|
|
222
|
-
return hfId;
|
|
223
|
-
}
|
|
33
|
+
const uid = uaidFull.match(/(?:^|;)uid=([^;]+)(?:;|$)/i)?.[1]?.trim();
|
|
34
|
+
if (uid && AGENTVERSE_ADDRESS_REGEX.test(uid)) {
|
|
35
|
+
return uid;
|
|
224
36
|
}
|
|
225
|
-
} catch {
|
|
226
|
-
}
|
|
227
|
-
try {
|
|
228
|
-
const headers = options.token ? { authorization: `Bearer ${options.token}` } : {};
|
|
229
|
-
const payload = await requestJson(
|
|
230
|
-
`${options.huggingFaceApiBase ?? HF_API_BASE}/models?search=${encodeURIComponent(normalized)}`,
|
|
231
|
-
{
|
|
232
|
-
fetch: context.fetch,
|
|
233
|
-
headers,
|
|
234
|
-
timeoutMs: context.timeoutMs,
|
|
235
|
-
signal: context.signal
|
|
236
|
-
}
|
|
237
|
-
);
|
|
238
|
-
if (Array.isArray(payload)) {
|
|
239
|
-
const match = payload.find((entry) => {
|
|
240
|
-
const id2 = asRecord(entry)?.id;
|
|
241
|
-
return typeof id2 === "string" && modelIdMatches(id2, normalized);
|
|
242
|
-
});
|
|
243
|
-
const id = asRecord(match)?.id;
|
|
244
|
-
if (typeof id === "string" && id.trim()) {
|
|
245
|
-
return id.trim();
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
} catch {
|
|
249
37
|
}
|
|
250
38
|
return null;
|
|
251
39
|
}
|
|
252
|
-
function
|
|
253
|
-
return
|
|
254
|
-
const url = typeof options.url === "function" ? options.url(subject) : options.url;
|
|
255
|
-
if (!url) {
|
|
256
|
-
return null;
|
|
257
|
-
}
|
|
258
|
-
const data = await requestJson(url, {
|
|
259
|
-
fetch: context.fetch,
|
|
260
|
-
timeoutMs: context.timeoutMs,
|
|
261
|
-
signal: context.signal
|
|
262
|
-
});
|
|
263
|
-
if (!data || !Array.isArray(data.rows)) {
|
|
264
|
-
return null;
|
|
265
|
-
}
|
|
266
|
-
return data;
|
|
267
|
-
};
|
|
40
|
+
function num(value) {
|
|
41
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
268
42
|
}
|
|
269
|
-
|
|
270
|
-
const
|
|
271
|
-
const
|
|
272
|
-
return match?.[1]?.trim() || null;
|
|
273
|
-
};
|
|
274
|
-
const buildBenchmarkCandidates = (normalizedId) => {
|
|
275
|
-
const candidates = /* @__PURE__ */ new Set([normalizedId, `${normalizedId}:free`]);
|
|
276
|
-
if (normalizedId.endsWith(":free")) {
|
|
277
|
-
candidates.add(normalizedId.slice(0, -":free".length));
|
|
278
|
-
}
|
|
279
|
-
return [...candidates];
|
|
280
|
-
};
|
|
281
|
-
function createOpenRouterBenchmarksSource(options = {}) {
|
|
282
|
-
const frontendBase = stripTrailingSlashes(
|
|
283
|
-
options.openrouterFrontendBaseUrl ?? OPENROUTER_FRONTEND_BASE
|
|
284
|
-
);
|
|
285
|
-
const apiBase = stripTrailingSlashes(
|
|
286
|
-
options.openrouterApiBaseUrl ?? OPENROUTER_API_BASE
|
|
287
|
-
);
|
|
288
|
-
const hfBase = stripTrailingSlashes(
|
|
289
|
-
options.huggingFaceApiBase ?? HF_API_BASE
|
|
290
|
-
);
|
|
291
|
-
const cacheTtlMs = options.cacheTtlMs ?? 24 * 60 * 60 * 1e3;
|
|
292
|
-
const preferHuggingFace = options.preferHuggingFace ?? true;
|
|
293
|
-
let cache = null;
|
|
294
|
-
const loadBenchmarks = async (context) => {
|
|
295
|
-
if (cache && Date.now() - cache.fetchedAt < cacheTtlMs) {
|
|
296
|
-
return cache;
|
|
297
|
-
}
|
|
298
|
-
const payload = await requestJson(`${frontendBase}/models/find?q=benchmarks`, {
|
|
299
|
-
fetch: context.fetch,
|
|
300
|
-
timeoutMs: context.timeoutMs,
|
|
301
|
-
signal: context.signal
|
|
302
|
-
});
|
|
303
|
-
const categories = payload.data?.categories;
|
|
304
|
-
if (!categories || typeof categories !== "object") {
|
|
305
|
-
return null;
|
|
306
|
-
}
|
|
307
|
-
const byModel = /* @__PURE__ */ new Map();
|
|
308
|
-
const categoryMaxRank = /* @__PURE__ */ new Map();
|
|
309
|
-
for (const [model, entries] of Object.entries(categories)) {
|
|
310
|
-
if (!Array.isArray(entries) || entries.length === 0) {
|
|
311
|
-
continue;
|
|
312
|
-
}
|
|
313
|
-
const normalizedModel = model.trim().toLowerCase();
|
|
314
|
-
if (!normalizedModel) {
|
|
315
|
-
continue;
|
|
316
|
-
}
|
|
317
|
-
const registerAlias = (alias) => {
|
|
318
|
-
const candidate = alias?.trim().toLowerCase() ?? "";
|
|
319
|
-
if (candidate && !byModel.has(candidate)) {
|
|
320
|
-
byModel.set(candidate, entries);
|
|
321
|
-
}
|
|
322
|
-
};
|
|
323
|
-
byModel.set(normalizedModel, entries);
|
|
324
|
-
const noFree = normalizedModel.endsWith(":free") ? normalizedModel.slice(0, -":free".length) : null;
|
|
325
|
-
registerAlias(noFree);
|
|
326
|
-
registerAlias(stripDateSuffix(normalizedModel));
|
|
327
|
-
registerAlias(noFree ? stripDateSuffix(noFree) : null);
|
|
328
|
-
for (const entry of entries) {
|
|
329
|
-
const category = typeof entry.category === "string" ? entry.category.trim().toLowerCase() : null;
|
|
330
|
-
const rank = parseNumber(entry.rank);
|
|
331
|
-
if (!category || rank === null) {
|
|
332
|
-
continue;
|
|
333
|
-
}
|
|
334
|
-
const currentMax = categoryMaxRank.get(category) ?? 0;
|
|
335
|
-
if (rank > currentMax) {
|
|
336
|
-
categoryMaxRank.set(category, rank);
|
|
337
|
-
}
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
cache = { fetchedAt: Date.now(), byModel, categoryMaxRank };
|
|
341
|
-
return cache;
|
|
342
|
-
};
|
|
343
|
-
return async (subject, context) => {
|
|
344
|
-
const modelId = resolveModelId(subject);
|
|
345
|
-
if (!modelId) {
|
|
346
|
-
return null;
|
|
347
|
-
}
|
|
348
|
-
const normalizedId = modelId.trim().toLowerCase();
|
|
349
|
-
let huggingFaceId = null;
|
|
350
|
-
if (preferHuggingFace) {
|
|
351
|
-
huggingFaceId = await resolveHuggingFaceModelId(modelId, context, {
|
|
352
|
-
openrouterApiBase: apiBase,
|
|
353
|
-
huggingFaceApiBase: hfBase
|
|
354
|
-
}).catch(() => null);
|
|
355
|
-
if (huggingFaceId) {
|
|
356
|
-
try {
|
|
357
|
-
const payload = await requestJson(
|
|
358
|
-
`${hfBase}/models/${encodeModelIdForPath(huggingFaceId)}`,
|
|
359
|
-
{
|
|
360
|
-
fetch: context.fetch,
|
|
361
|
-
timeoutMs: context.timeoutMs,
|
|
362
|
-
signal: context.signal
|
|
363
|
-
}
|
|
364
|
-
);
|
|
365
|
-
const metrics = extractModelIndexMetrics(payload);
|
|
366
|
-
if (metrics.length > 0) {
|
|
367
|
-
const score = metrics.reduce((sum, m) => sum + m.value, 0) / metrics.length;
|
|
368
|
-
return {
|
|
369
|
-
rows: [
|
|
370
|
-
{
|
|
371
|
-
model: modelId,
|
|
372
|
-
score,
|
|
373
|
-
weight: metrics.length
|
|
374
|
-
}
|
|
375
|
-
],
|
|
376
|
-
coverageWeight: metrics.length,
|
|
377
|
-
sources: [HUGGINGFACE_MODEL_INDEX_SOURCE],
|
|
378
|
-
huggingFaceId
|
|
379
|
-
};
|
|
380
|
-
}
|
|
381
|
-
} catch {
|
|
382
|
-
}
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
const snapshot = await loadBenchmarks(context);
|
|
386
|
-
if (!snapshot) {
|
|
387
|
-
return null;
|
|
388
|
-
}
|
|
389
|
-
let entries;
|
|
390
|
-
for (const candidate of buildBenchmarkCandidates(normalizedId)) {
|
|
391
|
-
entries = snapshot.byModel.get(candidate);
|
|
392
|
-
if (entries?.length) {
|
|
393
|
-
break;
|
|
394
|
-
}
|
|
395
|
-
}
|
|
396
|
-
if (!entries?.length) {
|
|
397
|
-
return null;
|
|
398
|
-
}
|
|
399
|
-
const rows = [];
|
|
400
|
-
for (const entry of entries) {
|
|
401
|
-
const category = typeof entry.category === "string" ? entry.category.trim().toLowerCase() : void 0;
|
|
402
|
-
const rank = parseNumber(entry.rank);
|
|
403
|
-
if (!category || rank === null) {
|
|
404
|
-
continue;
|
|
405
|
-
}
|
|
406
|
-
const maxRank = snapshot.categoryMaxRank.get(category);
|
|
407
|
-
if (!maxRank || maxRank <= 0) {
|
|
408
|
-
continue;
|
|
409
|
-
}
|
|
410
|
-
const volume = parseNumber(entry.volume);
|
|
411
|
-
const count = parseNumber(entry.count);
|
|
412
|
-
rows.push({
|
|
413
|
-
model: modelId,
|
|
414
|
-
category,
|
|
415
|
-
rank,
|
|
416
|
-
outOf: maxRank,
|
|
417
|
-
score: (maxRank - rank + 1) / maxRank * 100,
|
|
418
|
-
weight: volume && volume > 0 ? volume : count && count > 0 ? count : 1
|
|
419
|
-
});
|
|
420
|
-
}
|
|
421
|
-
if (rows.length === 0) {
|
|
422
|
-
return null;
|
|
423
|
-
}
|
|
424
|
-
return {
|
|
425
|
-
rows,
|
|
426
|
-
coverageWeight: rows.reduce((sum, row) => sum + (row.weight ?? 1), 0),
|
|
427
|
-
sources: [OPENROUTER_CATEGORY_SOURCE],
|
|
428
|
-
huggingFaceId: huggingFaceId ?? void 0
|
|
429
|
-
};
|
|
430
|
-
};
|
|
431
|
-
}
|
|
432
|
-
function createOpenRouterEvalsSignalAdapter(options = {}) {
|
|
433
|
-
const lowCoverageThreshold = options.lowCoverageThreshold ?? 5;
|
|
434
|
-
const minCategoryCount = options.minCategoryCount ?? 1;
|
|
435
|
-
const source = options.source ?? createOpenRouterBenchmarksSource();
|
|
43
|
+
function createAgentverseInsightsSignalAdapter(options = {}) {
|
|
44
|
+
const baseUrl = options.baseUrl ?? AGENTVERSE_API_BASE;
|
|
45
|
+
const fetchAgentStatus = options.fetchAgentStatus ?? true;
|
|
436
46
|
const collect = async (subject, context) => {
|
|
437
47
|
const now = context.now.toISOString();
|
|
438
|
-
const
|
|
439
|
-
|
|
440
|
-
try {
|
|
441
|
-
data = await source(subject, context);
|
|
442
|
-
} catch (error) {
|
|
443
|
-
if (error instanceof Hcs25CollectorHttpError && error.status === 404) {
|
|
444
|
-
data = null;
|
|
445
|
-
} else {
|
|
446
|
-
return [
|
|
447
|
-
{
|
|
448
|
-
signalId: "openrouter.evals",
|
|
449
|
-
status: statusFromError(error),
|
|
450
|
-
fields: [
|
|
451
|
-
{
|
|
452
|
-
scope: "additional",
|
|
453
|
-
values: {
|
|
454
|
-
openrouterEvalStatus: "error",
|
|
455
|
-
openrouterEvalUpdatedAt: now
|
|
456
|
-
}
|
|
457
|
-
}
|
|
458
|
-
]
|
|
459
|
-
}
|
|
460
|
-
];
|
|
461
|
-
}
|
|
462
|
-
}
|
|
463
|
-
const rows = data && modelId ? data.rows.filter((row) => modelIdMatches(row.model, modelId)) : [];
|
|
464
|
-
if (!data || rows.length === 0) {
|
|
48
|
+
const address = resolveAgentAddress(subject);
|
|
49
|
+
if (!address) {
|
|
465
50
|
return [
|
|
466
51
|
{
|
|
467
|
-
signalId: "
|
|
52
|
+
signalId: "agentverse.insights",
|
|
468
53
|
status: "missing",
|
|
469
54
|
fields: [
|
|
470
55
|
{
|
|
471
56
|
scope: "additional",
|
|
472
57
|
values: {
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
openrouterEvalMetricsCount: 0
|
|
58
|
+
agentverseInsightsStatus: "missing",
|
|
59
|
+
agentverseInsightsUpdatedAt: now
|
|
476
60
|
}
|
|
477
61
|
}
|
|
478
62
|
]
|
|
479
63
|
}
|
|
480
64
|
];
|
|
481
65
|
}
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
totalWeight += weight;
|
|
493
|
-
if (row.category) {
|
|
494
|
-
const existing = categoryStats.get(row.category) ?? {
|
|
495
|
-
scoreSum: 0,
|
|
496
|
-
weightSum: 0
|
|
497
|
-
};
|
|
498
|
-
existing.scoreSum += score2 * weight;
|
|
499
|
-
existing.weightSum += weight;
|
|
500
|
-
categoryStats.set(row.category, existing);
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
if (totalWeight <= 0) {
|
|
504
|
-
return [
|
|
505
|
-
{
|
|
506
|
-
signalId: "openrouter.evals",
|
|
507
|
-
status: "missing",
|
|
508
|
-
fields: [
|
|
66
|
+
if (!context.force) {
|
|
67
|
+
const additional = readSubjectAdditional(subject);
|
|
68
|
+
const storedStatus = readString(additional, "agentverseInsightsStatus");
|
|
69
|
+
const storedUpdatedMs = parseTimestampMs(
|
|
70
|
+
additional.agentverseInsightsUpdatedAt
|
|
71
|
+
);
|
|
72
|
+
if (storedStatus && storedUpdatedMs !== null) {
|
|
73
|
+
const ttl = storedStatus === "ok" ? options.ttlMs ?? 6 * 60 * 60 * 1e3 : options.failureTtlMs ?? 60 * 60 * 1e3;
|
|
74
|
+
if (context.now.getTime() - storedUpdatedMs < ttl) {
|
|
75
|
+
return [
|
|
509
76
|
{
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
77
|
+
signalId: "agentverse.insights",
|
|
78
|
+
status: storedStatus === "ok" ? "ok" : "missing",
|
|
79
|
+
provenance: {
|
|
80
|
+
source: "agentverse",
|
|
81
|
+
subjectId: address,
|
|
82
|
+
fetchedAt: new Date(storedUpdatedMs).toISOString()
|
|
514
83
|
}
|
|
515
84
|
}
|
|
516
|
-
]
|
|
85
|
+
];
|
|
517
86
|
}
|
|
518
|
-
|
|
87
|
+
}
|
|
519
88
|
}
|
|
520
|
-
const
|
|
521
|
-
const
|
|
522
|
-
const
|
|
523
|
-
const
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
89
|
+
const results = [];
|
|
90
|
+
const sources = [];
|
|
91
|
+
const contractParam = options.contract ?? readString(readSubjectAdditional(subject), "agentverseInsightsContract");
|
|
92
|
+
const contractsToTry = contractParam === "mainnet" || contractParam === "testnet" ? [contractParam] : ["mainnet", "testnet"];
|
|
93
|
+
let insightsResult = { status: "missing" };
|
|
94
|
+
for (const contract of contractsToTry) {
|
|
95
|
+
const insightsUrl = `${baseUrl}/v1/search/analytics/insights/${encodeURIComponent(address)}?contract=${contract}`;
|
|
96
|
+
try {
|
|
97
|
+
const data = await requestJson(
|
|
98
|
+
insightsUrl,
|
|
99
|
+
{
|
|
100
|
+
fetch: context.fetch,
|
|
101
|
+
timeoutMs: context.timeoutMs,
|
|
102
|
+
signal: context.signal
|
|
103
|
+
}
|
|
104
|
+
);
|
|
105
|
+
const hasAnySignal = num(data.rating) !== null || num(data.readme_quality_score) !== null || num(data.readme_uniqueness_score) !== null || num(data.interactions_score) !== null || num(data.avg_response_time) !== null || (num(data.asi1_total_interactions) ?? 0) > 0 || (num(data.asi1_recent_interactions) ?? 0) > 0 || (num(data.verifier_total_interactions) ?? 0) > 0 || (num(data.verifier_recent_interactions) ?? 0) > 0;
|
|
106
|
+
if (!hasAnySignal) {
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
insightsResult = { data, contract, url: insightsUrl };
|
|
110
|
+
break;
|
|
111
|
+
} catch (error) {
|
|
112
|
+
if (error instanceof Hcs25CollectorHttpError && (error.status === 404 || error.status === 422)) {
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
insightsResult = {
|
|
116
|
+
status: isTimeoutError(error) ? "timeout" : "upstream-error"
|
|
117
|
+
};
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
if ("data" in insightsResult) {
|
|
122
|
+
const { data, contract, url } = insightsResult;
|
|
123
|
+
sources.push("insights");
|
|
124
|
+
results.push({
|
|
125
|
+
signalId: "agentverse.insights",
|
|
531
126
|
status: "ok",
|
|
532
|
-
value:
|
|
127
|
+
value: num(data.rating),
|
|
533
128
|
fields: [
|
|
534
129
|
{
|
|
535
130
|
scope: "additional",
|
|
536
131
|
values: {
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
132
|
+
agentverseInsightsUpdatedAt: now,
|
|
133
|
+
agentverseInsightsStatus: "ok",
|
|
134
|
+
agentverseInsightsAddress: address,
|
|
135
|
+
agentverseInsightsContract: contract,
|
|
136
|
+
agentverseInsightsSources: [...INSIGHTS_SOURCES],
|
|
137
|
+
agentverseInsightsRating: num(data.rating),
|
|
138
|
+
agentverseInsightsReadmeQualityScore: num(
|
|
139
|
+
data.readme_quality_score
|
|
140
|
+
),
|
|
141
|
+
agentverseInsightsReadmeUniquenessScore: num(
|
|
142
|
+
data.readme_uniqueness_score
|
|
143
|
+
),
|
|
144
|
+
agentverseInsightsInteractionsScore: num(data.interactions_score),
|
|
145
|
+
agentverseInsightsAvgResponseTime: num(data.avg_response_time),
|
|
146
|
+
agentverseInsightsAsi1TotalInteractions: num(data.asi1_total_interactions) ?? 0,
|
|
147
|
+
agentverseInsightsAsi1TotalSuccessInteractions: num(data.asi1_total_success_interactions) ?? 0,
|
|
148
|
+
agentverseInsightsAsi1RecentInteractions: num(data.asi1_recent_interactions) ?? 0,
|
|
149
|
+
agentverseInsightsAsi1RecentSuccessInteractions: num(data.asi1_recent_success_interactions) ?? 0,
|
|
150
|
+
agentverseInsightsVerifierTotalInteractions: num(data.verifier_total_interactions) ?? 0,
|
|
151
|
+
agentverseInsightsVerifierTotalSuccessInteractions: num(data.verifier_total_success_interactions) ?? 0,
|
|
152
|
+
agentverseInsightsVerifierRecentInteractions: num(data.verifier_recent_interactions) ?? 0,
|
|
153
|
+
agentverseInsightsVerifierRecentSuccessInteractions: num(data.verifier_recent_success_interactions) ?? 0
|
|
546
154
|
}
|
|
547
155
|
}
|
|
548
156
|
],
|
|
549
157
|
provenance: {
|
|
550
|
-
source: "
|
|
551
|
-
|
|
158
|
+
source: "agentverse",
|
|
159
|
+
sourceUrl: url,
|
|
160
|
+
subjectId: address,
|
|
552
161
|
fetchedAt: now
|
|
553
162
|
}
|
|
554
|
-
}
|
|
555
|
-
];
|
|
556
|
-
};
|
|
557
|
-
return {
|
|
558
|
-
id: "openrouter-evals",
|
|
559
|
-
produces: ["openrouter.evals"],
|
|
560
|
-
timeoutMs: options.timeoutMs,
|
|
561
|
-
includeRegistries: options.includeRegistries ?? DEFAULT_MODEL_REGISTRIES,
|
|
562
|
-
appliesTo: options.appliesTo,
|
|
563
|
-
collect
|
|
564
|
-
};
|
|
565
|
-
}
|
|
566
|
-
function createJsonLeaderboardSource(options) {
|
|
567
|
-
const modelKey = options.modelKey ?? "model";
|
|
568
|
-
const scoreKey = options.scoreKey ?? "score";
|
|
569
|
-
const votesKey = options.votesKey ?? "votes";
|
|
570
|
-
return async (_subject, context) => {
|
|
571
|
-
const data = await requestJson(options.url, {
|
|
572
|
-
fetch: context.fetch,
|
|
573
|
-
timeoutMs: context.timeoutMs,
|
|
574
|
-
signal: context.signal
|
|
575
|
-
});
|
|
576
|
-
const record = asRecord(data);
|
|
577
|
-
const rows = Array.isArray(data) ? data : record?.entries ?? record?.rows ?? record?.leaderboard;
|
|
578
|
-
if (!Array.isArray(rows)) {
|
|
579
|
-
return null;
|
|
580
|
-
}
|
|
581
|
-
const entries = [];
|
|
582
|
-
for (const row of rows) {
|
|
583
|
-
const rowRecord = asRecord(row);
|
|
584
|
-
if (!rowRecord) {
|
|
585
|
-
continue;
|
|
586
|
-
}
|
|
587
|
-
const model = readString(rowRecord, modelKey);
|
|
588
|
-
const score = rowRecord[scoreKey];
|
|
589
|
-
const votes = rowRecord[votesKey];
|
|
590
|
-
if (!model || typeof score !== "number" || !Number.isFinite(score)) {
|
|
591
|
-
continue;
|
|
592
|
-
}
|
|
593
|
-
entries.push({
|
|
594
|
-
model,
|
|
595
|
-
score,
|
|
596
|
-
votes: typeof votes === "number" ? votes : void 0
|
|
597
163
|
});
|
|
598
|
-
}
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
let current = html;
|
|
604
|
-
for (; ; ) {
|
|
605
|
-
const next = current.replace(/<[^<>]*>/g, "");
|
|
606
|
-
if (next === current) {
|
|
607
|
-
return next;
|
|
608
|
-
}
|
|
609
|
-
current = next;
|
|
610
|
-
}
|
|
611
|
-
};
|
|
612
|
-
const decodeHtmlEntities = (html) => html.replace(/ /gi, " ").replace(/</gi, "<").replace(/>/gi, ">").replace(/"/gi, '"').replace(/'/g, "'").replace(/&#(\d+);/g, (_match, codepoint) => {
|
|
613
|
-
const parsed = Number(codepoint);
|
|
614
|
-
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
615
|
-
return "";
|
|
616
|
-
}
|
|
617
|
-
try {
|
|
618
|
-
return String.fromCodePoint(parsed);
|
|
619
|
-
} catch {
|
|
620
|
-
return "";
|
|
621
|
-
}
|
|
622
|
-
}).replace(/&/gi, "&");
|
|
623
|
-
const normalizeArenaLabel = (value) => {
|
|
624
|
-
const trimmed = value.trim();
|
|
625
|
-
const open = trimmed.lastIndexOf("(");
|
|
626
|
-
const base = open > 0 && trimmed.endsWith(")") ? trimmed.slice(0, open).trim() : trimmed;
|
|
627
|
-
const suffix = open > 0 && trimmed.endsWith(")") ? trimmed.slice(open + 1, -1).trim() : null;
|
|
628
|
-
const compactSuffix = suffix?.split(/\s+/).join("") ?? "";
|
|
629
|
-
const isDate = suffix !== null && (/^20\d{2}-\d{2}-\d{2}$/.test(suffix) || /^20\d{2}\d{2}\d{2}$/.test(compactSuffix));
|
|
630
|
-
const combined = !isDate && suffix ? `${base} ${suffix}` : base;
|
|
631
|
-
const slugged = combined.toLowerCase().replace(/[^a-z0-9]+/g, "-");
|
|
632
|
-
let start = 0;
|
|
633
|
-
let end = slugged.length;
|
|
634
|
-
while (start < end && slugged.charCodeAt(start) === 45) {
|
|
635
|
-
start += 1;
|
|
636
|
-
}
|
|
637
|
-
while (end > start && slugged.charCodeAt(end - 1) === 45) {
|
|
638
|
-
end -= 1;
|
|
639
|
-
}
|
|
640
|
-
return slugged.slice(start, end);
|
|
641
|
-
};
|
|
642
|
-
const normalizeAlphaNum = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, "");
|
|
643
|
-
const stripTrailingDateSuffix = (value) => {
|
|
644
|
-
const match = value.trim().toLowerCase().match(/^(.*?)-(20\d{2}-\d{2}-\d{2})$/) ?? value.trim().toLowerCase().match(/^(.*?)-(20\d{2}\d{2})$/);
|
|
645
|
-
return match?.[1]?.trim() || null;
|
|
646
|
-
};
|
|
647
|
-
function buildArenaMatchKeys(label, organization) {
|
|
648
|
-
const normalized = normalizeArenaLabel(label);
|
|
649
|
-
const keys = /* @__PURE__ */ new Set();
|
|
650
|
-
if (!normalized) {
|
|
651
|
-
return [];
|
|
652
|
-
}
|
|
653
|
-
const pushKey = (value) => {
|
|
654
|
-
const candidate = value?.trim() ?? "";
|
|
655
|
-
if (!candidate) {
|
|
656
|
-
return;
|
|
657
|
-
}
|
|
658
|
-
keys.add(candidate);
|
|
659
|
-
keys.add(normalizeAlphaNum(candidate));
|
|
660
|
-
};
|
|
661
|
-
const expandChatGptAliases = (value) => {
|
|
662
|
-
if (value.startsWith("chatgpt-")) {
|
|
663
|
-
pushKey(`gpt-${value.slice("chatgpt-".length)}`);
|
|
664
|
-
}
|
|
665
|
-
if (value.startsWith("gpt-")) {
|
|
666
|
-
const rest = value.slice("gpt-".length);
|
|
667
|
-
pushKey(`chatgpt-${rest}`);
|
|
668
|
-
pushKey(`chatgpt-${rest}-latest`);
|
|
669
|
-
}
|
|
670
|
-
};
|
|
671
|
-
const expandDateAliases = (value) => {
|
|
672
|
-
const stripped = stripTrailingDateSuffix(value);
|
|
673
|
-
if (stripped) {
|
|
674
|
-
pushKey(stripped);
|
|
675
|
-
expandChatGptAliases(stripped);
|
|
676
|
-
}
|
|
677
|
-
};
|
|
678
|
-
pushKey(normalized);
|
|
679
|
-
const org = normalizeArenaLabel(organization ?? "");
|
|
680
|
-
if (org) {
|
|
681
|
-
pushKey(`${org}-${normalized}`);
|
|
682
|
-
pushKey(`${org}/${normalized}`);
|
|
683
|
-
}
|
|
684
|
-
expandChatGptAliases(normalized);
|
|
685
|
-
expandDateAliases(normalized);
|
|
686
|
-
if (normalized.endsWith("-latest")) {
|
|
687
|
-
const without = normalized.slice(0, -"-latest".length);
|
|
688
|
-
pushKey(without);
|
|
689
|
-
expandChatGptAliases(without);
|
|
690
|
-
expandDateAliases(without);
|
|
691
|
-
}
|
|
692
|
-
return [...keys].filter(Boolean);
|
|
693
|
-
}
|
|
694
|
-
const arenaCandidateKeys = (modelId) => {
|
|
695
|
-
const base = modelId.trim().toLowerCase().replace(/:free$/i, "");
|
|
696
|
-
const candidates = /* @__PURE__ */ new Set();
|
|
697
|
-
const add = (value) => {
|
|
698
|
-
const trimmed = value.trim();
|
|
699
|
-
if (trimmed) {
|
|
700
|
-
candidates.add(normalizeArenaLabel(trimmed));
|
|
701
|
-
candidates.add(normalizeAlphaNum(trimmed));
|
|
702
|
-
}
|
|
703
|
-
};
|
|
704
|
-
add(base);
|
|
705
|
-
add(base.replace(/\//g, "-"));
|
|
706
|
-
const withoutFast = base.replace(/-fast$/i, "");
|
|
707
|
-
if (withoutFast !== base) {
|
|
708
|
-
add(withoutFast);
|
|
709
|
-
}
|
|
710
|
-
const stripped = stripTrailingDateSuffix(base);
|
|
711
|
-
if (stripped) {
|
|
712
|
-
add(stripped);
|
|
713
|
-
}
|
|
714
|
-
if (base.startsWith("gpt-")) {
|
|
715
|
-
add(`chatgpt-${base.slice(4)}`);
|
|
716
|
-
}
|
|
717
|
-
if (base.startsWith("chatgpt-")) {
|
|
718
|
-
add(`gpt-${base.slice(8)}`);
|
|
719
|
-
}
|
|
720
|
-
if (base.endsWith("-latest")) {
|
|
721
|
-
add(base.slice(0, -"-latest".length));
|
|
722
|
-
}
|
|
723
|
-
const orgPrefix = base.includes("/") ? base.split("/")[0] : null;
|
|
724
|
-
if (orgPrefix) {
|
|
725
|
-
const model = base.slice(orgPrefix.length + 1);
|
|
726
|
-
add(`${orgPrefix}-${model}`);
|
|
727
|
-
add(`${orgPrefix}/${model}`);
|
|
728
|
-
}
|
|
729
|
-
return [...candidates].filter(Boolean);
|
|
730
|
-
};
|
|
731
|
-
function createOpenLmArenaSource(options = {}) {
|
|
732
|
-
const url = options.url ?? OPENLM_ARENA_URL;
|
|
733
|
-
const cacheTtlMs = options.cacheTtlMs ?? 24 * 60 * 60 * 1e3;
|
|
734
|
-
let cache = null;
|
|
735
|
-
return async (_subject, context) => {
|
|
736
|
-
if (cache && Date.now() - cache.fetchedAt < cacheTtlMs) {
|
|
737
|
-
return cache.entries;
|
|
738
|
-
}
|
|
739
|
-
const html = await requestText(url, {
|
|
740
|
-
fetch: context.fetch,
|
|
741
|
-
timeoutMs: context.timeoutMs,
|
|
742
|
-
signal: context.signal,
|
|
743
|
-
headers: { accept: "text/html" }
|
|
744
|
-
});
|
|
745
|
-
const tableMatch = html.match(
|
|
746
|
-
/<table[^>]*class=sortable[^>]*>.*?<\/table>/is
|
|
747
|
-
);
|
|
748
|
-
if (!tableMatch) {
|
|
749
|
-
return null;
|
|
750
|
-
}
|
|
751
|
-
const tbodyMatch = tableMatch[0].match(/<tbody[^>]*>(.*?)<\/tbody>/is);
|
|
752
|
-
const tbody = tbodyMatch?.[1];
|
|
753
|
-
if (!tbody) {
|
|
754
|
-
return null;
|
|
755
|
-
}
|
|
756
|
-
const entries = [];
|
|
757
|
-
for (const rowMatch of tbody.matchAll(/<tr[^>]*>(.*?)<\/tr>/gis)) {
|
|
758
|
-
const rowHtml = rowMatch[1] ?? "";
|
|
759
|
-
if (!rowHtml) {
|
|
760
|
-
continue;
|
|
761
|
-
}
|
|
762
|
-
const cells = Array.from(rowHtml.matchAll(/<td[^>]*>(.*?)<\/td>/gis)).map(
|
|
763
|
-
(match) => match[1] ?? ""
|
|
764
|
-
);
|
|
765
|
-
const rawLabelCell = cells[1];
|
|
766
|
-
const rawScoreCell = cells[2];
|
|
767
|
-
const rawOrgCell = cells[8];
|
|
768
|
-
const modelLabel = decodeHtmlEntities(
|
|
769
|
-
stripHtmlTags(rawLabelCell ?? "")
|
|
770
|
-
).trim();
|
|
771
|
-
if (!modelLabel) {
|
|
772
|
-
continue;
|
|
773
|
-
}
|
|
774
|
-
const score = parseNumber(
|
|
775
|
-
decodeHtmlEntities(stripHtmlTags(rawScoreCell ?? "")).trim()
|
|
776
|
-
);
|
|
777
|
-
if (score === null) {
|
|
778
|
-
continue;
|
|
779
|
-
}
|
|
780
|
-
const organization = decodeHtmlEntities(stripHtmlTags(rawOrgCell ?? "")).trim() || void 0;
|
|
781
|
-
entries.push({ model: modelLabel, score, organization });
|
|
782
|
-
}
|
|
783
|
-
if (entries.length === 0) {
|
|
784
|
-
return null;
|
|
785
|
-
}
|
|
786
|
-
cache = { fetchedAt: Date.now(), entries };
|
|
787
|
-
return entries;
|
|
788
|
-
};
|
|
789
|
-
}
|
|
790
|
-
function createChatbotArenaSignalAdapter(options = {}) {
|
|
791
|
-
const source = options.source ?? createOpenLmArenaSource();
|
|
792
|
-
const collect = async (subject, context) => {
|
|
793
|
-
const now = context.now.toISOString();
|
|
794
|
-
const modelId = resolveModelId(subject);
|
|
795
|
-
const wanted = /* @__PURE__ */ new Set();
|
|
796
|
-
for (const id of [modelId, ...options.aliases ?? []]) {
|
|
797
|
-
if (!id) {
|
|
798
|
-
continue;
|
|
799
|
-
}
|
|
800
|
-
for (const key of arenaCandidateKeys(id)) {
|
|
801
|
-
wanted.add(key);
|
|
802
|
-
}
|
|
803
|
-
wanted.add(id.trim().toLowerCase());
|
|
804
|
-
}
|
|
805
|
-
let entries = null;
|
|
806
|
-
try {
|
|
807
|
-
entries = await source(subject, context);
|
|
808
|
-
} catch (error) {
|
|
809
|
-
return [
|
|
810
|
-
{
|
|
811
|
-
signalId: "chatbot-arena.eval",
|
|
812
|
-
status: statusFromError(error)
|
|
813
|
-
}
|
|
814
|
-
];
|
|
815
|
-
}
|
|
816
|
-
const entry = entries?.find((candidate) => {
|
|
817
|
-
if (wanted.has(candidate.model.trim().toLowerCase()) || [...wanted].some((id) => modelIdMatches(candidate.model, id))) {
|
|
818
|
-
return true;
|
|
819
|
-
}
|
|
820
|
-
const keys = buildArenaMatchKeys(
|
|
821
|
-
candidate.model,
|
|
822
|
-
candidate.organization
|
|
823
|
-
);
|
|
824
|
-
return keys.some((key) => wanted.has(key));
|
|
825
|
-
}) ?? null;
|
|
826
|
-
if (!entries || entries.length === 0 || !entry) {
|
|
827
|
-
return [
|
|
828
|
-
{
|
|
829
|
-
signalId: "chatbot-arena.eval",
|
|
830
|
-
status: "missing",
|
|
831
|
-
fields: [
|
|
832
|
-
{
|
|
833
|
-
scope: "additional",
|
|
834
|
-
values: {
|
|
835
|
-
chatbotArenaEvalStatus: "missing",
|
|
836
|
-
chatbotArenaEvalUpdatedAt: now
|
|
837
|
-
}
|
|
838
|
-
}
|
|
839
|
-
]
|
|
840
|
-
}
|
|
841
|
-
];
|
|
842
|
-
}
|
|
843
|
-
const scores = entries.map((e) => e.score);
|
|
844
|
-
const min = Math.min(...scores);
|
|
845
|
-
const max = Math.max(...scores);
|
|
846
|
-
const normalized = max > min ? 100 * (entry.score - min) / (max - min) : 50;
|
|
847
|
-
return [
|
|
848
|
-
{
|
|
849
|
-
signalId: "chatbot-arena.eval",
|
|
850
|
-
status: "ok",
|
|
851
|
-
value: Math.round(normalized * 100) / 100,
|
|
164
|
+
} else {
|
|
165
|
+
const stored = insightsResult.status === "missing" ? "missing" : "upstream-error";
|
|
166
|
+
results.push({
|
|
167
|
+
signalId: "agentverse.insights",
|
|
168
|
+
status: insightsResult.status === "timeout" ? "timeout" : stored === "missing" ? "missing" : "error",
|
|
852
169
|
fields: [
|
|
853
170
|
{
|
|
854
171
|
scope: "additional",
|
|
855
172
|
values: {
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
chatbotArenaEvalUpdatedAt: now,
|
|
861
|
-
chatbotArenaEvalSources: [CHATBOT_ARENA_SOURCE]
|
|
173
|
+
agentverseInsightsUpdatedAt: now,
|
|
174
|
+
agentverseInsightsStatus: stored,
|
|
175
|
+
agentverseInsightsAddress: address,
|
|
176
|
+
agentverseInsightsSources: [...INSIGHTS_SOURCES]
|
|
862
177
|
}
|
|
863
178
|
}
|
|
864
179
|
],
|
|
865
180
|
provenance: {
|
|
866
|
-
source: "
|
|
867
|
-
subjectId:
|
|
181
|
+
source: "agentverse",
|
|
182
|
+
subjectId: address,
|
|
868
183
|
fetchedAt: now
|
|
869
184
|
}
|
|
870
|
-
}
|
|
871
|
-
];
|
|
872
|
-
};
|
|
873
|
-
return {
|
|
874
|
-
id: "chatbot-arena",
|
|
875
|
-
produces: ["chatbot-arena.eval"],
|
|
876
|
-
timeoutMs: options.timeoutMs,
|
|
877
|
-
includeRegistries: options.includeRegistries ?? DEFAULT_MODEL_REGISTRIES,
|
|
878
|
-
appliesTo: options.appliesTo,
|
|
879
|
-
collect
|
|
880
|
-
};
|
|
881
|
-
}
|
|
882
|
-
function createHuggingFaceSignalAdapter(options = {}) {
|
|
883
|
-
const apiBase = stripTrailingSlashes(options.apiBase ?? HF_API_BASE);
|
|
884
|
-
const collect = async (subject, context) => {
|
|
885
|
-
const now = context.now.toISOString();
|
|
886
|
-
const additional = readSubjectAdditional(subject);
|
|
887
|
-
const storedHfId = readString(additional, "huggingFaceModelId") ?? readString(additional, "openrouterHuggingFaceId");
|
|
888
|
-
const modelId = resolveModelId(subject);
|
|
889
|
-
if (!modelId && !storedHfId) {
|
|
890
|
-
return [
|
|
891
|
-
{
|
|
892
|
-
signalId: "huggingface.model_index",
|
|
893
|
-
status: "missing",
|
|
894
|
-
fields: [
|
|
895
|
-
{
|
|
896
|
-
scope: "additional",
|
|
897
|
-
values: {
|
|
898
|
-
huggingFaceEvalStatus: "missing",
|
|
899
|
-
huggingFaceEvalUpdatedAt: now
|
|
900
|
-
}
|
|
901
|
-
}
|
|
902
|
-
]
|
|
903
|
-
}
|
|
904
|
-
];
|
|
185
|
+
});
|
|
905
186
|
}
|
|
906
|
-
|
|
907
|
-
const
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
if (!storedHfId && options.resolveHuggingFaceId !== false && modelId) {
|
|
911
|
-
const resolved = await resolveHuggingFaceModelId(modelId, context, {
|
|
912
|
-
huggingFaceApiBase: apiBase,
|
|
913
|
-
openrouterApiBase: options.openrouterApiBase,
|
|
914
|
-
token: options.token
|
|
915
|
-
}).catch(() => null);
|
|
916
|
-
if (resolved) {
|
|
917
|
-
huggingFaceId = resolved;
|
|
918
|
-
resolvedViaOpenRouter = true;
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
const data = await requestJson(
|
|
922
|
-
`${apiBase}/models/${encodeModelIdForPath(huggingFaceId)}`,
|
|
923
|
-
{
|
|
187
|
+
if (fetchAgentStatus) {
|
|
188
|
+
const agentUrl = `${baseUrl}/v1/search/agents/${encodeURIComponent(address)}${contractParam ? `?contract=${contractParam}` : ""}`;
|
|
189
|
+
try {
|
|
190
|
+
const agent = await requestJson(agentUrl, {
|
|
924
191
|
fetch: context.fetch,
|
|
925
|
-
headers,
|
|
926
192
|
timeoutMs: context.timeoutMs,
|
|
927
193
|
signal: context.signal
|
|
928
|
-
}
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
const popularityScore = computePopularityScore(
|
|
935
|
-
downloads,
|
|
936
|
-
likes,
|
|
937
|
-
options.downloadsCap ?? 1e7,
|
|
938
|
-
options.likesCap ?? 1e5
|
|
939
|
-
);
|
|
940
|
-
let score = null;
|
|
941
|
-
let mode;
|
|
942
|
-
if (modelIndexScore !== null && popularityScore !== null) {
|
|
943
|
-
score = modelIndexScore * 0.8 + popularityScore * 0.2;
|
|
944
|
-
mode = "mixed";
|
|
945
|
-
} else if (modelIndexScore !== null) {
|
|
946
|
-
score = modelIndexScore;
|
|
947
|
-
mode = "model-index";
|
|
948
|
-
} else if (popularityScore !== null) {
|
|
949
|
-
score = popularityScore;
|
|
950
|
-
mode = "popularity";
|
|
951
|
-
} else {
|
|
952
|
-
mode = "missing";
|
|
953
|
-
}
|
|
954
|
-
const normalized = score === null ? null : Math.round(Math.min(100, Math.max(0, score)) * 100) / 100;
|
|
955
|
-
const sources = [];
|
|
956
|
-
if (modelIndexScore !== null) {
|
|
957
|
-
sources.push(HUGGINGFACE_MODEL_INDEX_SOURCE);
|
|
958
|
-
}
|
|
959
|
-
if (popularityScore !== null) {
|
|
960
|
-
sources.push(HUGGINGFACE_POPULARITY_SOURCE);
|
|
961
|
-
}
|
|
962
|
-
return [
|
|
963
|
-
{
|
|
964
|
-
signalId: "huggingface.model_index",
|
|
965
|
-
status: normalized === null ? "missing" : "ok",
|
|
966
|
-
value: normalized,
|
|
194
|
+
});
|
|
195
|
+
sources.push("search-agent");
|
|
196
|
+
results.push({
|
|
197
|
+
signalId: "agentverse.agent_status",
|
|
198
|
+
status: "ok",
|
|
199
|
+
value: agent.status ?? null,
|
|
967
200
|
fields: [
|
|
968
201
|
{
|
|
969
|
-
scope: "
|
|
202
|
+
scope: "metrics",
|
|
970
203
|
values: {
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
huggingFaceSignalScore: normalized,
|
|
974
|
-
huggingFaceEvalMetricsCount: metrics.length,
|
|
975
|
-
huggingFaceDownloads: downloads,
|
|
976
|
-
huggingFaceLikes: likes,
|
|
977
|
-
huggingFaceEvalMode: mode,
|
|
978
|
-
huggingFaceEvalStatus: normalized === null ? "missing" : "ok",
|
|
979
|
-
huggingFaceEvalUpdatedAt: now,
|
|
980
|
-
huggingFaceEvalSources: sources,
|
|
981
|
-
...resolvedViaOpenRouter ? { openrouterHuggingFaceId: huggingFaceId } : {}
|
|
204
|
+
isOnline: agent.status === "active",
|
|
205
|
+
lastActiveAt: agent.last_updated ?? null
|
|
982
206
|
}
|
|
983
207
|
}
|
|
984
208
|
],
|
|
985
209
|
provenance: {
|
|
986
|
-
source: "
|
|
987
|
-
sourceUrl:
|
|
988
|
-
subjectId:
|
|
210
|
+
source: "agentverse",
|
|
211
|
+
sourceUrl: agentUrl,
|
|
212
|
+
subjectId: address,
|
|
989
213
|
fetchedAt: now
|
|
990
214
|
}
|
|
991
|
-
}
|
|
992
|
-
];
|
|
993
|
-
} catch (error) {
|
|
994
|
-
const missing = error instanceof Hcs25CollectorHttpError && error.status === 404;
|
|
995
|
-
return [
|
|
996
|
-
{
|
|
997
|
-
signalId: "huggingface.model_index",
|
|
998
|
-
status: missing ? "missing" : statusFromError(error),
|
|
999
|
-
fields: [
|
|
1000
|
-
{
|
|
1001
|
-
scope: "additional",
|
|
1002
|
-
values: {
|
|
1003
|
-
huggingFaceEvalStatus: missing ? "missing" : "error",
|
|
1004
|
-
huggingFaceEvalUpdatedAt: now
|
|
1005
|
-
}
|
|
1006
|
-
}
|
|
1007
|
-
]
|
|
1008
|
-
}
|
|
1009
|
-
];
|
|
1010
|
-
}
|
|
1011
|
-
};
|
|
1012
|
-
return {
|
|
1013
|
-
id: "huggingface-model-index",
|
|
1014
|
-
produces: ["huggingface.model_index"],
|
|
1015
|
-
timeoutMs: options.timeoutMs,
|
|
1016
|
-
includeRegistries: options.includeRegistries ?? DEFAULT_MODEL_REGISTRIES,
|
|
1017
|
-
appliesTo: options.appliesTo,
|
|
1018
|
-
collect
|
|
1019
|
-
};
|
|
1020
|
-
}
|
|
1021
|
-
function createOpenLlmJsonSource(options) {
|
|
1022
|
-
const modelKey = options.modelKey ?? "model";
|
|
1023
|
-
return async (_subject, context) => {
|
|
1024
|
-
const data = await requestJson(options.url, {
|
|
1025
|
-
fetch: context.fetch,
|
|
1026
|
-
timeoutMs: context.timeoutMs,
|
|
1027
|
-
signal: context.signal
|
|
1028
|
-
});
|
|
1029
|
-
const record = asRecord(data);
|
|
1030
|
-
const rows = Array.isArray(data) ? data : record?.rows;
|
|
1031
|
-
if (!Array.isArray(rows)) {
|
|
1032
|
-
return null;
|
|
1033
|
-
}
|
|
1034
|
-
const out = [];
|
|
1035
|
-
for (const row of rows) {
|
|
1036
|
-
const rowRecord = asRecord(row);
|
|
1037
|
-
if (!rowRecord) {
|
|
1038
|
-
continue;
|
|
1039
|
-
}
|
|
1040
|
-
const model = readString(rowRecord, modelKey);
|
|
1041
|
-
if (!model) {
|
|
1042
|
-
continue;
|
|
1043
|
-
}
|
|
1044
|
-
const metrics = {};
|
|
1045
|
-
for (const [key, value] of Object.entries(rowRecord)) {
|
|
1046
|
-
const parsed = parseNumber(value);
|
|
1047
|
-
if (key !== modelKey && parsed !== null) {
|
|
1048
|
-
metrics[key] = parsed;
|
|
1049
|
-
}
|
|
1050
|
-
}
|
|
1051
|
-
out.push({ model, metrics });
|
|
1052
|
-
}
|
|
1053
|
-
return out;
|
|
1054
|
-
};
|
|
1055
|
-
}
|
|
1056
|
-
const tokenizeForSimilarity = (value) => new Set(
|
|
1057
|
-
value.toLowerCase().split(/[^a-z0-9]+/).filter((token) => token.length > 1)
|
|
1058
|
-
);
|
|
1059
|
-
const jaccardSimilarity = (a, b) => {
|
|
1060
|
-
if (a.size === 0 || b.size === 0) {
|
|
1061
|
-
return 0;
|
|
1062
|
-
}
|
|
1063
|
-
let intersection = 0;
|
|
1064
|
-
for (const token of a) {
|
|
1065
|
-
if (b.has(token)) {
|
|
1066
|
-
intersection += 1;
|
|
1067
|
-
}
|
|
1068
|
-
}
|
|
1069
|
-
return intersection / (a.size + b.size - intersection);
|
|
1070
|
-
};
|
|
1071
|
-
function createOpenLlmHuggingFaceSource(options = {}) {
|
|
1072
|
-
const apiBase = stripTrailingSlashes(
|
|
1073
|
-
options.huggingFaceApiBase ?? HF_API_BASE
|
|
1074
|
-
);
|
|
1075
|
-
const datasetsBase = stripTrailingSlashes(
|
|
1076
|
-
options.huggingFaceDatasetsBase ?? HF_DATASETS_BASE
|
|
1077
|
-
);
|
|
1078
|
-
const datasetId = options.datasetId ?? OPENLLM_DATASET_ID;
|
|
1079
|
-
const branch = options.datasetBranch ?? OPENLLM_DATASET_BRANCH;
|
|
1080
|
-
const cacheTtlMs = options.cacheTtlMs ?? 24 * 60 * 60 * 1e3;
|
|
1081
|
-
const minSimilarity = options.minSimilarity ?? 0.6;
|
|
1082
|
-
const pathCache = /* @__PURE__ */ new Map();
|
|
1083
|
-
const orgCache = /* @__PURE__ */ new Map();
|
|
1084
|
-
const listTree = async (context, path) => {
|
|
1085
|
-
const url = `${apiBase}/datasets/${datasetId}/tree/${branch}/${encodeModelIdForPath(path)}`;
|
|
1086
|
-
const payload = await requestJson(url, {
|
|
1087
|
-
fetch: context.fetch,
|
|
1088
|
-
timeoutMs: context.timeoutMs,
|
|
1089
|
-
signal: context.signal
|
|
1090
|
-
});
|
|
1091
|
-
return Array.isArray(payload) ? payload : [];
|
|
1092
|
-
};
|
|
1093
|
-
const latestResultsPath = async (context, dir) => {
|
|
1094
|
-
const candidates = (await listTree(context, dir)).map((entry) => typeof entry.path === "string" ? entry.path : "").filter((path) => path.includes("results_") && path.endsWith(".json"));
|
|
1095
|
-
return candidates.length > 0 ? candidates.sort().at(-1) ?? null : null;
|
|
1096
|
-
};
|
|
1097
|
-
const resolveViaOrg = async (context, huggingFaceId) => {
|
|
1098
|
-
const slashIndex = huggingFaceId.indexOf("/");
|
|
1099
|
-
if (slashIndex <= 0 || slashIndex === huggingFaceId.length - 1) {
|
|
1100
|
-
return null;
|
|
1101
|
-
}
|
|
1102
|
-
const org = huggingFaceId.slice(0, slashIndex);
|
|
1103
|
-
const repo = huggingFaceId.slice(slashIndex + 1);
|
|
1104
|
-
if (!org || !repo) {
|
|
1105
|
-
return null;
|
|
1106
|
-
}
|
|
1107
|
-
const orgKey = org.toLowerCase();
|
|
1108
|
-
let paths;
|
|
1109
|
-
const cached = orgCache.get(orgKey);
|
|
1110
|
-
if (cached && Date.now() - cached.fetchedAt < cacheTtlMs) {
|
|
1111
|
-
paths = cached.paths;
|
|
1112
|
-
} else {
|
|
1113
|
-
try {
|
|
1114
|
-
paths = (await listTree(context, org)).map((entry) => typeof entry.path === "string" ? entry.path : "").filter(
|
|
1115
|
-
(path) => path.startsWith(`${org}/`) && !path.slice(org.length + 1).includes("/") && path.slice(org.length + 1).length > 0
|
|
1116
|
-
);
|
|
215
|
+
});
|
|
1117
216
|
} catch {
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
if (paths.length === 0) {
|
|
1123
|
-
return null;
|
|
1124
|
-
}
|
|
1125
|
-
const targetTokens = tokenizeForSimilarity(repo);
|
|
1126
|
-
let bestPath = null;
|
|
1127
|
-
let bestSimilarity = 0;
|
|
1128
|
-
for (const path of paths) {
|
|
1129
|
-
const candidateRepo = path.slice(org.length + 1);
|
|
1130
|
-
const similarity = jaccardSimilarity(
|
|
1131
|
-
targetTokens,
|
|
1132
|
-
tokenizeForSimilarity(candidateRepo)
|
|
1133
|
-
);
|
|
1134
|
-
if (similarity > bestSimilarity) {
|
|
1135
|
-
bestSimilarity = similarity;
|
|
1136
|
-
bestPath = path;
|
|
217
|
+
results.push({
|
|
218
|
+
signalId: "agentverse.agent_status",
|
|
219
|
+
status: "missing"
|
|
220
|
+
});
|
|
1137
221
|
}
|
|
1138
222
|
}
|
|
1139
|
-
if (
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
} catch {
|
|
1145
|
-
return null;
|
|
1146
|
-
}
|
|
1147
|
-
};
|
|
1148
|
-
const resolveResultsPath = async (context, huggingFaceId) => {
|
|
1149
|
-
const key = huggingFaceId.toLowerCase();
|
|
1150
|
-
const cached = pathCache.get(key);
|
|
1151
|
-
if (cached && Date.now() - cached.fetchedAt < cacheTtlMs) {
|
|
1152
|
-
return cached.path;
|
|
1153
|
-
}
|
|
1154
|
-
let path = null;
|
|
1155
|
-
try {
|
|
1156
|
-
path = await latestResultsPath(context, huggingFaceId);
|
|
1157
|
-
} catch {
|
|
1158
|
-
path = null;
|
|
1159
|
-
}
|
|
1160
|
-
if (!path) {
|
|
1161
|
-
path = await resolveViaOrg(context, huggingFaceId);
|
|
1162
|
-
}
|
|
1163
|
-
pathCache.set(key, { path, fetchedAt: Date.now() });
|
|
1164
|
-
return path;
|
|
1165
|
-
};
|
|
1166
|
-
return async (subject, context) => {
|
|
1167
|
-
const additional = readSubjectAdditional(subject);
|
|
1168
|
-
const huggingFaceId = readString(additional, "huggingFaceModelId") ?? readString(additional, "openrouterHuggingFaceId") ?? resolveModelId(subject);
|
|
1169
|
-
if (!huggingFaceId) {
|
|
1170
|
-
return null;
|
|
1171
|
-
}
|
|
1172
|
-
const resultPath = await resolveResultsPath(context, huggingFaceId);
|
|
1173
|
-
if (!resultPath) {
|
|
1174
|
-
return null;
|
|
1175
|
-
}
|
|
1176
|
-
const url = `${datasetsBase}/${datasetId}/resolve/${branch}/${resultPath}`;
|
|
1177
|
-
const payload = await requestJson(url, {
|
|
1178
|
-
fetch: context.fetch,
|
|
1179
|
-
timeoutMs: context.timeoutMs,
|
|
1180
|
-
signal: context.signal
|
|
1181
|
-
});
|
|
1182
|
-
const results = asRecord(payload)?.results;
|
|
1183
|
-
if (!isJsonObject(results)) {
|
|
1184
|
-
return null;
|
|
1185
|
-
}
|
|
1186
|
-
const metrics = {};
|
|
1187
|
-
for (const groupValue of Object.values(
|
|
1188
|
-
results
|
|
1189
|
-
)) {
|
|
1190
|
-
if (!isJsonObject(groupValue)) {
|
|
1191
|
-
continue;
|
|
1192
|
-
}
|
|
1193
|
-
for (const [name, rawValue] of Object.entries(
|
|
1194
|
-
groupValue
|
|
1195
|
-
)) {
|
|
1196
|
-
const normalizedName = name.trim().toLowerCase();
|
|
1197
|
-
if (!normalizedName || normalizedName.includes("stderr") || normalizedName === "alias") {
|
|
1198
|
-
continue;
|
|
1199
|
-
}
|
|
1200
|
-
const value = parseNumber(rawValue);
|
|
1201
|
-
if (value === null || !shouldIncludeMetric(normalizedName)) {
|
|
1202
|
-
continue;
|
|
1203
|
-
}
|
|
1204
|
-
metrics[name] = value;
|
|
1205
|
-
}
|
|
1206
|
-
}
|
|
1207
|
-
if (Object.keys(metrics).length === 0) {
|
|
1208
|
-
return null;
|
|
1209
|
-
}
|
|
1210
|
-
return [{ model: huggingFaceId, metrics }];
|
|
1211
|
-
};
|
|
1212
|
-
}
|
|
1213
|
-
function createOpenLlmSignalAdapter(options = {}) {
|
|
1214
|
-
const source = options.source ?? createOpenLlmHuggingFaceSource();
|
|
1215
|
-
const collect = async (subject, context) => {
|
|
1216
|
-
const now = context.now.toISOString();
|
|
1217
|
-
const modelId = resolveModelId(subject);
|
|
1218
|
-
if (!modelId) {
|
|
1219
|
-
return [
|
|
1220
|
-
{
|
|
1221
|
-
signalId: "openllm.leaderboard",
|
|
1222
|
-
status: "missing",
|
|
1223
|
-
fields: [
|
|
1224
|
-
{
|
|
1225
|
-
scope: "additional",
|
|
1226
|
-
values: {
|
|
1227
|
-
openLlmEvalStatus: "missing",
|
|
1228
|
-
openLlmEvalUpdatedAt: now
|
|
1229
|
-
}
|
|
1230
|
-
}
|
|
1231
|
-
]
|
|
1232
|
-
}
|
|
1233
|
-
];
|
|
1234
|
-
}
|
|
1235
|
-
let rows = null;
|
|
1236
|
-
try {
|
|
1237
|
-
rows = await source(subject, context);
|
|
1238
|
-
} catch (error) {
|
|
1239
|
-
return [
|
|
1240
|
-
{
|
|
1241
|
-
signalId: "openllm.leaderboard",
|
|
1242
|
-
status: statusFromError(error),
|
|
1243
|
-
fields: [
|
|
1244
|
-
{
|
|
1245
|
-
scope: "additional",
|
|
1246
|
-
values: {
|
|
1247
|
-
openLlmEvalStatus: "error",
|
|
1248
|
-
openLlmEvalUpdatedAt: now
|
|
1249
|
-
}
|
|
1250
|
-
}
|
|
1251
|
-
]
|
|
1252
|
-
}
|
|
1253
|
-
];
|
|
1254
|
-
}
|
|
1255
|
-
const row = rows?.find((candidate) => modelIdMatches(candidate.model, modelId)) ?? (rows?.length === 1 ? rows[0] : null);
|
|
1256
|
-
if (!row) {
|
|
1257
|
-
return [
|
|
1258
|
-
{
|
|
1259
|
-
signalId: "openllm.leaderboard",
|
|
1260
|
-
status: "missing",
|
|
1261
|
-
fields: [
|
|
1262
|
-
{
|
|
1263
|
-
scope: "additional",
|
|
1264
|
-
values: {
|
|
1265
|
-
openLlmEvalStatus: "missing",
|
|
1266
|
-
openLlmEvalUpdatedAt: now
|
|
1267
|
-
}
|
|
1268
|
-
}
|
|
1269
|
-
]
|
|
1270
|
-
}
|
|
1271
|
-
];
|
|
1272
|
-
}
|
|
1273
|
-
const values = Object.values(row.metrics).map(normalizeMetricValue).filter((value) => value !== null);
|
|
1274
|
-
const score = values.length > 0 ? Math.round(
|
|
1275
|
-
values.reduce((sum, v) => sum + v, 0) / values.length * 100
|
|
1276
|
-
) / 100 : null;
|
|
1277
|
-
return [
|
|
1278
|
-
{
|
|
1279
|
-
signalId: "openllm.leaderboard",
|
|
1280
|
-
status: score === null ? "missing" : "ok",
|
|
1281
|
-
value: score,
|
|
223
|
+
if (sources.length > 0) {
|
|
224
|
+
results.push({
|
|
225
|
+
signalId: "agentverse.sources",
|
|
226
|
+
status: "ok",
|
|
227
|
+
value: sources,
|
|
1282
228
|
fields: [
|
|
1283
229
|
{
|
|
1284
230
|
scope: "additional",
|
|
1285
|
-
values: {
|
|
1286
|
-
openLlmEvalScore: score,
|
|
1287
|
-
openLlmEvalMetricsCount: values.length,
|
|
1288
|
-
openLlmEvalStatus: score === null ? "missing" : "ok",
|
|
1289
|
-
openLlmEvalUpdatedAt: now,
|
|
1290
|
-
openLlmEvalSources: [...OPENLLM_EVAL_SOURCES]
|
|
1291
|
-
}
|
|
231
|
+
values: { agentverseInsightsSources: sources }
|
|
1292
232
|
}
|
|
1293
|
-
]
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
fetchedAt: now
|
|
1298
|
-
}
|
|
1299
|
-
}
|
|
1300
|
-
];
|
|
233
|
+
]
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
return results;
|
|
1301
237
|
};
|
|
1302
238
|
return {
|
|
1303
|
-
id: "
|
|
1304
|
-
produces: [
|
|
239
|
+
id: "agentverse-insights",
|
|
240
|
+
produces: [
|
|
241
|
+
"agentverse.insights",
|
|
242
|
+
"agentverse.agent_status",
|
|
243
|
+
"agentverse.sources"
|
|
244
|
+
],
|
|
1305
245
|
timeoutMs: options.timeoutMs,
|
|
1306
|
-
includeRegistries: options.includeRegistries ??
|
|
246
|
+
includeRegistries: options.includeRegistries ?? DEFAULT_INCLUDED_REGISTRIES,
|
|
247
|
+
excludeRegistries: options.excludeRegistries,
|
|
1307
248
|
appliesTo: options.appliesTo,
|
|
1308
249
|
collect
|
|
1309
250
|
};
|
|
1310
251
|
}
|
|
1311
252
|
export {
|
|
1312
|
-
|
|
1313
|
-
computePopularityScore,
|
|
1314
|
-
createChatbotArenaSignalAdapter,
|
|
1315
|
-
createHuggingFaceSignalAdapter,
|
|
1316
|
-
createJsonLeaderboardSource,
|
|
1317
|
-
createOpenLlmHuggingFaceSource,
|
|
1318
|
-
createOpenLlmJsonSource,
|
|
1319
|
-
createOpenLlmSignalAdapter,
|
|
1320
|
-
createOpenLmArenaSource,
|
|
1321
|
-
createOpenRouterBenchmarksSource,
|
|
1322
|
-
createOpenRouterEvalsHttpSource,
|
|
1323
|
-
createOpenRouterEvalsSignalAdapter,
|
|
1324
|
-
extractModelIndexMetrics,
|
|
1325
|
-
modelIdMatches,
|
|
1326
|
-
normalizeLogScore,
|
|
1327
|
-
normalizeMetricValue,
|
|
1328
|
-
resolveHuggingFaceModelId,
|
|
1329
|
-
resolveModelId,
|
|
1330
|
-
shouldIncludeMetric
|
|
253
|
+
createAgentverseInsightsSignalAdapter
|
|
1331
254
|
};
|
|
1332
255
|
//# sourceMappingURL=standards-sdk.es137.js.map
|