@llmindset/hf-mcp 0.1.21 → 0.1.23

package/dist/docs-search/docs-semantic-search.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"docs-semantic-search.test.js","sourceRoot":"","sources":["../../src/docs-search/docs-semantic-search.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGlD,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAC1B,MAAM,CAAC,KAAK,GAAG,SAAyB,CAAC;AAEzC,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,IAAI,aAA4B,CAAC;IAEjC,UAAU,CAAC,GAAG,EAAE;QACf,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;QACpC,EAAE,CAAC,aAAa,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC9E,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aAC/B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;YACpE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACrE,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aAC/B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YACpF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,wDAAwD;oBAC9D,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,kFAAkF;oBACnG,iBAAiB,EAAE,0BAA0B;oBAC7C,QAAQ,EAAE,yBAAyB;iBACnC;gBACD;oBACC,IAAI,EAAE,sCAAsC;oBAC5C,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,kFAAkF;oBACnG,iBAAiB,EAAE,0BAA0B;oBAC7C,QAAQ,EAAE,gBAAgB;iBAC1B;gBACD;oBACC,IAAI,EAAE,2EAA2E;oBACjF,OAAO,EAAE,gBAAgB;oBACzB,QAAQ,EAAE,YAAY;oBACtB,eAAe,EAAE,mEAAmE;oBACpF,iBAAiB,EAAE,aAAa;iBAChC;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;YAGlE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wDAAwD,CAAC,CAAC;YACnF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAG5C,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC;YAC/D,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC;YACpF,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;YAClD,MAAM,CAAC,QAAQ,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,kBAAkB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAG/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,yCAAyC,CAAC,CAAC;YACpE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,oDAAoD,CAAC,CAAC;YAG/E,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CACvB,wFAAwF,CACxF,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,uFAAuF,CAAC,CAAC;YAGlH,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,yDAAyD,CAAC,CAAC;YACpF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,gDAAgD,CAAC,CAAC;YAG3E,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wDAAwD,CAAC,CAAC;YACnF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sCAAsC,CAAC,CAAC;YACjE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,2EAA2E,CAAC,CAAC;YAGtG,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,WAAW,GAAG,gBAAgB,CAAC,IAAI,GAAG,8CAA8C,CAAC,CAAC;QAChH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,wCAAwC;oBAC9C,OAAO,EAAE,cAAc;oBACvB,QAAQ,EAAE,cAAc;oBACxB,eAAe,EAAE,gDAAgD;oBACjE,iBAAiB,EAAE,cAAc;iBACjC;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;YAGrE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wCAAwC,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YACnE,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,wDAAwD;oBAC9D,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,sBAAsB;oBAChC,eAAe,EAAE,sCAAsC;oBACvD,iBAAiB,EAAE,MAAM;oBACzB,QAAQ,EAAE,yBAAyB;iBACnC;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YAGhE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,yDAAyD,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,mFAAmF;oBACzF,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,2CAA2C;oBAC5D,iBAAiB,EAAE,WAAW;iBAC9B;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAG7D,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC3E,MAAM,aAAa,GAAG;gBAErB;oBACC,IAAI,EAAE,gCAAgC;oBACtC,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,uCAAuC;oBACxD,iBAAiB,EAAE,QAAQ;iBAC3B;gBACD;oBACC,IAAI,EAAE,iCAAiC;oBACvC,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,uCAAuC;oBACxD,iBAAiB,EAAE,QAAQ;iBAC3B;gBACD;oBACC,IAAI,EAAE,wBAAwB;oBAC9B,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,uCAAuC;oBACxD,iBAAiB,EAAE,QAAQ;iBAC3B;gBAED;oBACC,IAAI,EAAE,0BAA0B;oBAChC,OAAO,EAAE,cAAc;oBACvB,QAAQ,EAAE,mBAAmB;oBAC7B,eAAe,EAAE,gDAAgD;oBACjE,iBAAiB,EAAE,cAAc;iBACjC;gBAED;oBACC,IAAI,EAAE,sBAAsB;oBAC5B,OAAO,EAAE,UAAU;oBACnB,QAAQ,EAAE,eAAe;oBACzB,eAAe,EAAE,4CAA4C;oBAC7D,iBAAiB,EAAE,UAAU;iBAC7B;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAG7D,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC;YAC/D,MAAM,iBAAiB,GAAG,MAAM,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC;YACjF,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;YAEzE,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,iBAAiB,CAAC,CAAC;YACjD,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;YAG7C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,yCAAyC,CAAC,CAAC;YAGpE,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;YAC3E,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;YAC3E,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YAG5C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,8EAA8E,CAAC,CAAC;QAC1G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACxE,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aAC/B,CAAC,CAAC;YAEH,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE9D,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yEAAyE,EAAE,KAAK,IAAI,EAAE;YACxF,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,6BAA6B;oBACnC,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,oDAAoD;oBACrE,iBAAiB,EAAE,gBAAgB;oBACnC,QAAQ,EAAE,WAAW;iBACrB;gBACD;oBACC,IAAI,EAAE,8BAA8B;oBACpC,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,oDAAoD;oBACrE,iBAAiB,EAAE,gBAAgB;oBACnC,QAAQ,EAAE,WAAW;iBACrB;gBACD;oBACC,IAAI,EAAE,6BAA6B;oBACnC,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,oDAAoD;oBACrE,iBAAiB,EAAE,gBAAgB;oBACnC,QAAQ,EAAE,WAAW;iBACrB;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;YAGlE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,qFAAqF,CAAC,CAAC;YAGhH,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,8BAA8B,CAAC,CAAC;YACzD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;YAGxD,MAAM,gBAAgB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YAC1E,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACpD,SAAS,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAE5D,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;QAC1G,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,2BAA2B;oBAC5C,iBAAiB,EAAE,QAAQ;iBAC3B;gBACD;oBACC,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,2BAA2B;oBAC5C,iBAAiB,EAAE,QAAQ;iBAC3B;gBACD;oBACC,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,cAAc;oBACvB,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,2BAA2B;oBAC5C,iBAAiB,EAAE,QAAQ;iBAC3B;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAG7D,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sCAAsC,CAAC,CAAC;YAGjE,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAGhD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAEhE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
+{"version":3,"file":"docs-semantic-search.test.js","sourceRoot":"","sources":["../../src/docs-search/docs-semantic-search.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGlD,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAC1B,MAAM,CAAC,KAAK,GAAG,SAAyB,CAAC;AAEzC,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,IAAI,aAA4B,CAAC;IAEjC,UAAU,CAAC,GAAG,EAAE;QACf,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;QACpC,EAAE,CAAC,aAAa,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC9E,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aAC/B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;YACpE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACrE,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aAC/B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YACpF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,wDAAwD;oBAC9D,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,kFAAkF;oBACnG,iBAAiB,EAAE,0BAA0B;oBAC7C,QAAQ,EAAE,yBAAyB;iBACnC;gBACD;oBACC,IAAI,EAAE,sCAAsC;oBAC5C,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,kFAAkF;oBACnG,iBAAiB,EAAE,0BAA0B;oBAC7C,QAAQ,EAAE,gBAAgB;iBAC1B;gBACD;oBACC,IAAI,EAAE,2EAA2E;oBACjF,OAAO,EAAE,gBAAgB;oBACzB,QAAQ,EAAE,YAAY;oBACtB,eAAe,EAAE,mEAAmE;oBACpF,iBAAiB,EAAE,aAAa;iBAChC;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;YAGlE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wDAAwD,CAAC,CAAC;YACnF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAG5C,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC;YAC/D,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC;YACpF,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;YAClD,MAAM,CAAC,QAAQ,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,kBAAkB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAG/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,yCAAyC,CAAC,CAAC;YACpE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,oDAAoD,CAAC,CAAC;YAG/E,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CACvB,wFAAwF,CACxF,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,uFAAuF,CAAC,CAAC;YAGlH,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,yDAAyD,CAAC,CAAC;YACpF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,gDAAgD,CAAC,CAAC;YAG3E,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wDAAwD,CAAC,CAAC;YACnF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sCAAsC,CAAC,CAAC;YACjE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,2EAA2E,CAAC,CAAC;YAGtG,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,WAAW,GAAG,gBAAgB,CAAC,IAAI,GAAG,8CAA8C,CAAC,CAAC;QAChH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,wCAAwC;oBAC9C,OAAO,EAAE,cAAc;oBACvB,QAAQ,EAAE,cAAc;oBACxB,eAAe,EAAE,gDAAgD;oBACjE,iBAAiB,EAAE,cAAc;iBACjC;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;YAGrE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wCAAwC,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YACnE,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,wDAAwD;oBAC9D,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,sBAAsB;oBAChC,eAAe,EAAE,sCAAsC;oBACvD,iBAAiB,EAAE,MAAM;oBACzB,QAAQ,EAAE,yBAAyB;iBACnC;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YAGhE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,yDAAyD,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,mFAAmF;oBACzF,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,2CAA2C;oBAC5D,iBAAiB,EAAE,WAAW;iBAC9B;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAG7D,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC3E,MAAM,aAAa,GAAG;gBAErB;oBACC,IAAI,EAAE,gCAAgC;oBACtC,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,uCAAuC;oBACxD,iBAAiB,EAAE,QAAQ;iBAC3B;gBACD;oBACC,IAAI,EAAE,iCAAiC;oBACvC,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,uCAAuC;oBACxD,iBAAiB,EAAE,QAAQ;iBAC3B;gBACD;oBACC,IAAI,EAAE,wBAAwB;oBAC9B,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,uCAAuC;oBACxD,iBAAiB,EAAE,QAAQ;iBAC3B;gBAED;oBACC,IAAI,EAAE,0BAA0B;oBAChC,OAAO,EAAE,cAAc;oBACvB,QAAQ,EAAE,mBAAmB;oBAC7B,eAAe,EAAE,gDAAgD;oBACjE,iBAAiB,EAAE,cAAc;iBACjC;gBAED;oBACC,IAAI,EAAE,sBAAsB;oBAC5B,OAAO,EAAE,UAAU;oBACnB,QAAQ,EAAE,eAAe;oBACzB,eAAe,EAAE,4CAA4C;oBAC7D,iBAAiB,EAAE,UAAU;iBAC7B;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAG7D,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC;YAC/D,MAAM,iBAAiB,GAAG,MAAM,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC;YACjF,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;YAEzE,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,iBAAiB,CAAC,CAAC;YACjD,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;YAG7C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,yCAAyC,CAAC,CAAC;YAGpE,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;YAC3E,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;YAC3E,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YAG5C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,8EAA8E,CAAC,CAAC;QAC1G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACxE,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aAC/B,CAAC,CAAC;YAEH,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE9D,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yEAAyE,EAAE,KAAK,IAAI,EAAE;YACxF,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,6BAA6B;oBACnC,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,oDAAoD;oBACrE,iBAAiB,EAAE,gBAAgB;oBACnC,QAAQ,EAAE,WAAW;iBACrB;gBACD;oBACC,IAAI,EAAE,8BAA8B;oBACpC,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,oDAAoD;oBACrE,iBAAiB,EAAE,gBAAgB;oBACnC,QAAQ,EAAE,WAAW;iBACrB;gBACD;oBACC,IAAI,EAAE,6BAA6B;oBACnC,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,WAAW;oBACrB,eAAe,EAAE,oDAAoD;oBACrE,iBAAiB,EAAE,gBAAgB;oBACnC,QAAQ,EAAE,WAAW;iBACrB;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;YAGlE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,qFAAqF,CAAC,CAAC;YAGhH,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,8BAA8B,CAAC,CAAC;YACzD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;YAGxD,MAAM,gBAAgB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YAC1E,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACpD,SAAS,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAE5D,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;QAC1G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAE/E,MAAM,eAAe,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;YAGtE,MAAM,QAAQ,GAAG,yCAAyC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAGvE,MAAM,WAAW,GAOZ,EAAE,CAAC;YAER,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,WAAW,CAAC,IAAI,CAAC;oBAChB,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ,CAAC,EAAE;oBACrB,eAAe,EAAE,uCAAuC,CAAC,EAAE;oBAC3D,iBAAiB,EAAE,QAAQ,CAAC,EAAE;oBAC9B,QAAQ,EAAE,WAAW,CAAC,EAAE;iBACxB,CAAC,CAAC;YACJ,CAAC;YAED,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;aACxC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAG/D,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,mDAAmD,CAAC,CAAC;YAC9E,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAG5C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,uCAAuC,CAAC,CAAC;YAGlE,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;YAGjE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,2CAA2C,CAAC,CAAC;YACtE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,uCAAuC,CAAC,CAAC;YAGlE,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,6BAA6B,gBAAgB,CAAC,IAAI,yCAAyC,CAAC,CAAC;YACnI,MAAM,oBAAoB,GAAG,MAAM,CAAC,QAAQ,CAAC,mCAAmC,CAAC,CAAC;YAIlF,MAAM,CAAC,aAAa,IAAI,oBAAoB,CAAC,CAAC,UAAU,EAAE,CAAC;YAG3D,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,gBAAgB,CAAC,IAAI,8CAA8C,CAAC,CAAC;YAG1G,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,aAAa,GAAG;gBACrB;oBACC,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,2BAA2B;oBAC5C,iBAAiB,EAAE,QAAQ;iBAC3B;gBACD;oBACC,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,2BAA2B;oBAC5C,iBAAiB,EAAE,QAAQ;iBAC3B;gBACD;oBACC,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,cAAc;oBACvB,QAAQ,EAAE,QAAQ;oBAClB,eAAe,EAAE,2BAA2B;oBAC5C,iBAAiB,EAAE,QAAQ;iBAC3B;aACD,CAAC;YAEF,SAAS,CAAC,qBAAqB,CAAC;gBAC/B,EAAE,EAAE,IAAI;gBACR,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAG7D,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sCAAsC,CAAC,CAAC;YAGjE,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAGhD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAEhE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}

package/dist/utilities.d.ts

@@ -5,4 +5,5 @@ export declare function formatUnknownDate(date: Date | string | undefined): stri
 export declare function formatNumber(num: number): string;
 export declare function formatBytes(bytes: number): string;
 export declare function escapeMarkdown(text: string): string;
+export declare function estimateTokens(text: string): number;
 //# sourceMappingURL=utilities.d.ts.map

package/dist/utilities.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utilities.d.ts","sourceRoot":"","sources":["../src/utilities.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,eAAe,QAE4E,CAAC;AAEzG,eAAO,MAAM,qBAAqB,QACsE,CAAC;AAGzG,wBAAgB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,CAGtD;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAUrF;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOhD;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CASjD;AAOD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAanD"}
+{"version":3,"file":"utilities.d.ts","sourceRoot":"","sources":["../src/utilities.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,eAAe,QAE4E,CAAC;AAEzG,eAAO,MAAM,qBAAqB,QACsE,CAAC;AAGzG,wBAAgB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,CAGtD;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAUrF;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOhD;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CASjD;AAOD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAanD;AAWD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD"}

package/dist/utilities.js

@@ -50,4 +50,8 @@ export function escapeMarkdown(text) {
         .replace(/>/g, '\\>')
         .replace(/#/g, '\\#');
 }
+const CHARS_PER_TOKEN = 3.3;
+export function estimateTokens(text) {
+    return Math.ceil(text.length / CHARS_PER_TOKEN);
+}
 //# sourceMappingURL=utilities.js.map

package/dist/utilities.js.map

@@ -1 +1 @@
-{"version":3,"file":"utilities.js","sourceRoot":"","sources":["../src/utilities.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAC3B,2GAA2G;IAC3G,uGAAuG,CAAC;AAEzG,MAAM,CAAC,MAAM,qBAAqB,GACjC,qFAAqF,GAAG,eAAe,CAAC;AAGzG,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC7C,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACvC,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAA+B;IAChE,IAAI,SAAS,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,YAAY,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7D,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAAE,OAAO,SAAS,CAAC;IAE/C,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAEnC,OAAO,GAAG,GAAG,CAAC,QAAQ,EAAE,IAAI,KAAK,KAAK,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACvC,IAAI,GAAG,IAAI,OAAO,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACzC,CAAC;SAAM,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACtC,CAAC;IACD,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACxC,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;QACzB,OAAO,GAAG,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAChD,CAAC;SAAM,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7C,CAAC;SAAM,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;AACpC,CAAC;AAOD,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAGrB,OAAO,IAAI;SACT,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;SACpB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;SACpB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;SACpB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;SACpB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACxB,CAAC"}
+{"version":3,"file":"utilities.js","sourceRoot":"","sources":["../src/utilities.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAC3B,2GAA2G;IAC3G,uGAAuG,CAAC;AAEzG,MAAM,CAAC,MAAM,qBAAqB,GACjC,qFAAqF,GAAG,eAAe,CAAC;AAGzG,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC7C,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACvC,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAA+B;IAChE,IAAI,SAAS,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,YAAY,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7D,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAAE,OAAO,SAAS,CAAC;IAE/C,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAEnC,OAAO,GAAG,GAAG,CAAC,QAAQ,EAAE,IAAI,KAAK,KAAK,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACvC,IAAI,GAAG,IAAI,OAAO,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACzC,CAAC;SAAM,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACtC,CAAC;IACD,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACxC,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;QACzB,OAAO,GAAG,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAChD,CAAC;SAAM,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7C,CAAC;SAAM,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;AACpC,CAAC;AAOD,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAGrB,OAAO,IAAI;SACT,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;SACpB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;SACpB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;SACpB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;SACpB,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACxB,CAAC;AAGD,MAAM,eAAe,GAAG,GAAG,CAAC;AAQ5B,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC;AACjD,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llmindset/hf-mcp",
-  "version": "0.1.21",
+  "version": "0.1.23",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -24,11 +24,13 @@
   },
   "dependencies": {
     "@huggingface/hub": "^2.1.0",
+    "turndown": "^7.2.0",
     "zod": "^3.24.4"
   },
   "devDependencies": {
     "@eslint/js": "^9.25.0",
     "@types/node": "^22.15.18",
+    "@types/turndown": "^5.0.5",
     "eslint": "^9.25.0",
     "globals": "^16.0.0",
     "rimraf": "^6.0.1",

package/src/docs-search/doc-fetch.test.ts

@@ -1,74 +1,105 @@
-import { describe, it, expect } from 'vitest';
+import { describe, it, expect, vi } from 'vitest';
 import { DocFetchTool } from './doc-fetch.js';
 
 describe('DocFetchTool', () => {
 	const tool = new DocFetchTool();
 
-	describe('URL conversion', () => {
-		it('should convert basic HF docs URL to GitHub URL', () => {
-			const hfUrl = 'https://huggingface.co/docs/dataset-viewer/index';
-			const expectedUrl =
-				'https://raw.githubusercontent.com/huggingface/dataset-viewer/refs/heads/main/docs/source/index.md';
-
-			const result = tool.processUrl(hfUrl);
-			expect(result).toBe(expectedUrl);
-		});
-
-		it('should handle URLs with fragments by removing them', () => {
-			const hfUrl = 'https://huggingface.co/docs/huggingface_hub/guides/upload#faster-uploads';
-			const expectedUrl =
-				'https://raw.githubusercontent.com/huggingface/huggingface_hub/refs/heads/main/docs/source/en/guides/upload.md';
-
-			const result = tool.processUrl(hfUrl);
-			expect(result).toBe(expectedUrl);
-		});
-
-		it('should handle transformers documentation', () => {
-			const hfUrl = 'https://huggingface.co/docs/transformers/model_doc/bert';
-			const expectedUrl =
-				'https://raw.githubusercontent.com/huggingface/transformers/refs/heads/main/docs/source/en/model_doc/bert.md';
-
-			const result = tool.processUrl(hfUrl);
-			expect(result).toBe(expectedUrl);
+	describe('URL validation', () => {
+		it('should accept valid HF docs URLs', () => {
+			const validUrls = [
+				'https://huggingface.co/docs/dataset-viewer/index',
+				'https://huggingface.co/docs/huggingface_hub/guides/upload#faster-uploads',
+				'https://huggingface.co/docs/transformers/model_doc/bert',
+				'https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion',
+				'https://huggingface.co/docs/timm/models',
+				'https://huggingface.co/docs/transformers',
+			];
+
+			for (const url of validUrls) {
+				expect(() => tool.validateUrl(url)).not.toThrow();
+			}
 		});
 
-		it('should handle diffusers documentation', () => {
-			const hfUrl = 'https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion';
-			const expectedUrl =
-				'https://raw.githubusercontent.com/huggingface/diffusers/refs/heads/main/docs/source/en/api/pipelines/stable_diffusion.md';
-
-			const result = tool.processUrl(hfUrl);
-			expect(result).toBe(expectedUrl);
+		it('should throw error for URLs not starting with correct prefix', () => {
+			const invalidUrls = [
+				'https://example.com/docs/something',
+				'https://github.com/huggingface/transformers',
+				'http://huggingface.co/docs/transformers',
+				'huggingface.co/docs/transformers',
+				'https://huggingface.co/models/bert-base-uncased',
+			];
+
+			for (const url of invalidUrls) {
+				expect(() => tool.validateUrl(url)).toThrow('That was not a valid Hugging Face document URL');
+			}
 		});
+	});
 
-		it('should handle timm documentation with custom package name', () => {
-			const hfUrl = 'https://huggingface.co/docs/timm/models';
-			const expectedUrl =
-				'https://raw.githubusercontent.com/huggingface/pytorch-image-models/refs/heads/main/hfdocs/source/models.md';
-
-			const result = tool.processUrl(hfUrl);
-			expect(result).toBe(expectedUrl);
+	describe('document chunking', () => {
+		it('should return small documents without chunking', async () => {
+			
+			// Mock fetch to return HTML that converts to short markdown
+			global.fetch = vi.fn().mockResolvedValue({
+				ok: true,
+				text: () => Promise.resolve('<h1>Short Document</h1><p>This is a short document.</p>'),
+			});
+
+			const result = await tool.fetch({ doc_url: 'https://huggingface.co/docs/test' });
+			
+			expect(result).toContain('# Short Document');
+			expect(result).toContain('This is a short document');
+			expect(result).not.toContain('DOCUMENT TRUNCATED');
 		});
 
-		it('should default to index.md when no path is provided', () => {
-			const hfUrl = 'https://huggingface.co/docs/transformers';
-			const expectedUrl =
-				'https://raw.githubusercontent.com/huggingface/transformers/refs/heads/main/docs/source/en/index.md';
-
-			const result = tool.processUrl(hfUrl);
-			expect(result).toBe(expectedUrl);
+		it('should chunk large documents and show truncation message', async () => {
+			// Mock fetch to return HTML that converts to long markdown
+			const longHtml = '<h1>Long Document</h1>' + '<p>This is a very long sentence that will be repeated many times to create a document that exceeds the 7500 token limit for testing chunking functionality.</p>'.repeat(200);
+			
+			global.fetch = vi.fn().mockResolvedValue({
+				ok: true,
+				text: () => Promise.resolve(longHtml),
+			});
+
+			const result = await tool.fetch({ doc_url: 'https://huggingface.co/docs/test' });
+			
+			expect(result).toContain('# Long Document');
+			expect(result).toContain('DOCUMENT TRUNCATED');
+			expect(result).toContain('CALL hf_doc_fetch WITH AN OFFSET OF');
 		});
 
-		it('should throw error for URLs not starting with correct prefix', () => {
-			const invalidUrl = 'https://example.com/docs/something';
-
-			expect(() => tool.processUrl(invalidUrl)).toThrow('That was not a valid Hugging Face document URL');
+		it('should return subsequent chunks with offset', async () => {
+			// Mock fetch to return the same long HTML
+			const longHtml = '<h1>Long Document</h1>' + '<p>This is a very long sentence that will be repeated many times to create a document that exceeds the 7500 token limit for testing chunking functionality.</p>'.repeat(200);
+			
+			global.fetch = vi.fn().mockResolvedValue({
+				ok: true,
+				text: () => Promise.resolve(longHtml),
+			});
+
+			// Get first chunk
+			const firstChunk = await tool.fetch({ doc_url: 'https://huggingface.co/docs/test' });
+			
+			// Extract offset from truncation message
+			const offsetMatch = firstChunk.match(/OFFSET OF (\d+)/);
+			expect(offsetMatch).toBeTruthy();
+			const offset = parseInt(offsetMatch?.[1] || '0', 10);
+
+			// Get second chunk
+			const secondChunk = await tool.fetch({ doc_url: 'https://huggingface.co/docs/test', offset });
+			
+			expect(secondChunk).not.toEqual(firstChunk);
+			expect(secondChunk.length).toBeGreaterThan(0);
 		});
 
-		it('should throw error for unknown package names', () => {
-			const unknownPackageUrl = 'https://huggingface.co/docs/unknown-package/guide';
+		it('should handle offset beyond document length', async () => {
+			global.fetch = vi.fn().mockResolvedValue({
+				ok: true,
+				text: () => Promise.resolve('<h1>Short Document</h1><p>This is short.</p>'),
+			});
 
-			expect(tool.processUrl(unknownPackageUrl)).toBe('https://huggingface.co/docs/unknown-package/guide');
+			const result = await tool.fetch({ doc_url: 'https://huggingface.co/docs/test', offset: 10000 });
+			
+			expect(result).toContain('Error: Offset 10000 is beyond');
 		});
 	});
-});
+});

package/src/docs-search/doc-fetch.ts

@@ -1,15 +1,21 @@
 import { z } from 'zod';
-import { DOC_MAPPINGS } from './doc-mappings.js';
+import TurndownService from 'turndown';
+import { estimateTokens } from '../utilities.js';
 
 export const DOC_FETCH_CONFIG = {
 	name: 'hf_doc_fetch',
-	description: 'Fetch a document from the Hugging Face documentation library.',
+	description: 'Fetch a document from the Hugging Face documentation library. For large documents, use offset to get subsequent chunks.',
 	schema: z.object({
 		doc_url: z
 			.string()
 			.min(28, 'Url should start with https://huggingface.co/docs/')
 			.max(200, 'Query too long')
 			.describe('Hugging Face documentation URL'),
+		offset: z
+			.number()
+			.min(0)
+			.optional()
+			.describe('Token offset for large documents (use the offset from truncation message)'),
 	}),
 	annotations: {
 		title: 'Fetch a document from the Hugging Face library',
@@ -21,72 +27,87 @@ export const DOC_FETCH_CONFIG = {
 
 export type DocFetchParams = z.infer<typeof DOC_FETCH_CONFIG.schema>;
 
-/**
- * Return a (usually) markdown document for a Doc URL
- *
- * There were a few different options for this...
- * 1) Return the HTML directly. Good option would include any post-processing (but prefer markdown)
- * 2) Get the HTML and convert to markdown
- * 3) Get the HTML and follow the GitHub link to get the raw markdown file
- * 4) Use a mapping.
- *
- * There is also the risk that the GitHub HEAD is newer than the indexed version. Will leave as is for now
- * but might want to reconsider this. Mappings were built from:
- * https://github.com/huggingface/doc-builder/blob/main/.github/workflows/build_embeddings.yml
- *
- * Falls back to HTML if no mapping is found...
- *
- */
 export class DocFetchTool {
+	private turndownService: TurndownService;
+
+	constructor() {
+		this.turndownService = new TurndownService({
+			headingStyle: 'atx',
+			codeBlockStyle: 'fenced',
+		});
+	}
+
 	/**
-	 * Convert HF docs URL to GitHub raw content URL
+	 * Validate HF docs URL
 	 */
-	processUrl(hfUrl: string): string {
-		// Validate URL format
+	validateUrl(hfUrl: string): void {
 		if (!hfUrl.startsWith('https://huggingface.co/docs/')) {
 			throw new Error('That was not a valid Hugging Face document URL');
 		}
-
-		// Remove the base URL and any fragment identifier
-		const urlPath = hfUrl.replace('https://huggingface.co/docs/', '').split('#')[0] || '';
-
-		// Extract package name and path
-		const parts = urlPath.split('/');
-		const packageName = parts[0] || '';
-
-		// Check if package exists in mapping
-		let fetchUrl = hfUrl;
-		const mapping = DOC_MAPPINGS[packageName];
-		if (mapping) {
-			const remainingPath = parts.slice(1).join('/');
-			const filePath = remainingPath ? `${remainingPath}.md` : 'index.md';
-
-			// Construct GitHub raw URL
-			const githubUrl = `https://raw.githubusercontent.com/${mapping.repo_id}/refs/heads/main/${mapping.doc_folder}/${filePath}`;
-			fetchUrl = githubUrl;
-		}
-
-		// Build the file path
-		return fetchUrl;
 	}
 
 	/**
-	 * Fetch markdown content from GitHub
+	 * Fetch content from Hugging Face docs URL and convert HTML to Markdown
 	 */
-	async fetch(url: string): Promise<string> {
+	async fetch(params: DocFetchParams): Promise<string> {
 		try {
-			const githubUrl = this.processUrl(url);
+			this.validateUrl(params.doc_url);
 
-			const response = await fetch(githubUrl);
+			const response = await fetch(params.doc_url);
 
 			if (!response.ok) {
 				throw new Error(`Failed to fetch document: ${response.status} ${response.statusText}`);
 			}
 
-			const content = await response.text();
-			return content;
+			const htmlContent = await response.text();
+			
+			// Convert HTML to Markdown
+			const fullMarkdownContent = this.turndownService.turndown(htmlContent);
+			
+			// Apply chunking logic
+			return this.applyChunking(fullMarkdownContent, params.offset || 0);
 		} catch (error) {
 			throw new Error(`Failed to fetch document: ${error instanceof Error ? error.message : 'Unknown error'}`);
 		}
 	}
+
+	/**
+	 * Apply chunking logic to markdown content
+	 */
+	private applyChunking(markdownContent: string, offset: number): string {
+		const totalTokens = estimateTokens(markdownContent);
+		const maxTokensPerChunk = 7500;
+		
+		// Calculate character positions based on tokens
+		const totalChars = markdownContent.length;
+		const charsPerToken = totalChars / totalTokens;
+		const startChar = Math.floor(offset * charsPerToken);
+		
+		// If offset is beyond document, return error message
+		if (startChar >= totalChars) {
+			return `Error: Offset ${offset} is beyond the document length (${totalTokens} tokens total).`;
+		}
+		
+		// If document is small enough and no offset, return as-is
+		if (totalTokens <= maxTokensPerChunk && offset === 0) {
+			return markdownContent;
+		}
+		
+		const maxCharsPerChunk = Math.floor(maxTokensPerChunk * charsPerToken);
+		const endChar = Math.min(startChar + maxCharsPerChunk, totalChars);
+		const chunk = markdownContent.slice(startChar, endChar);
+		
+		// Calculate next offset
+		const nextOffset = offset + estimateTokens(chunk);
+		const hasMore = nextOffset < totalTokens;
+		
+		let result = chunk;
+		
+		// Add truncation message if there's more content
+		if (hasMore) {
+			result += `\n\n=== DOCUMENT TRUNCATED. CALL ${DOC_FETCH_CONFIG.name} WITH AN OFFSET OF ${nextOffset} FOR THE NEXT CHUNK ===`;
+		}
+		
+		return result;
+	}
 }

package/src/docs-search/docs-semantic-search.test.ts

@@ -312,6 +312,71 @@ describe('DocSearchTool', () => {
 
 			await expect(docSearchTool.search({ query: 'test' })).rejects.toThrow('Failed to search documentation:');
 		});
+
+		it('should handle token budget by truncating or switching to links', async () => {
+			// Create a tool with a small token budget to trigger limits
+			const smallBudgetTool = new DocSearchTool(undefined, undefined, 5000); // 5k token budget
+			
+			// Create a result with a very long excerpt
+			const longText = 'This is a very long text that repeats. '.repeat(100); // ~3900 chars per excerpt
+			
+			// Create many results to exceed budget (5k tokens = ~16.5k chars)
+			const manyResults: Array<{
+				text: string;
+				product: string;
+				heading1: string;
+				source_page_url: string;
+				source_page_title: string;
+				heading2: string;
+			}> = [];
+			// Create enough to exceed token budget when formatted
+			for (let i = 0; i < 8; i++) {
+				manyResults.push({
+					text: longText,
+					product: 'hub',
+					heading1: `Page ${i}`,
+					source_page_url: `https://huggingface.co/docs/hub/page${i}`,
+					source_page_title: `Page ${i}`,
+					heading2: `Section ${i}`,
+				});
+			}
+
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				json: () => Promise.resolve(manyResults),
+			});
+
+			const result = await smallBudgetTool.search({ query: 'test' });
+
+			// Should contain the header
+			expect(result).toContain('# Documentation Library Search Results for "test"');
+			expect(result).toContain('Found 8 results');
+
+			// Early results should have full content
+			expect(result).toContain('This is a very long text that repeats');
+
+			// Debug: let's see what the result looks like
+			console.log('Result length:', result.length);
+			console.log('Estimated tokens:', Math.ceil(result.length / 3.3));
+			
+			// Check that early pages have content 
+			expect(result).toContain('#### Excerpt from the "Section 0" section');
+			expect(result).toContain('This is a very long text that repeats');
+			
+			// With token budget, we should see either truncation or link-only section
+			const hasTruncation = result.includes(`*[Content truncated - use ${DOC_FETCH_CONFIG.name} for full text or narrow search terms]*`);
+			const hasAdditionalResults = result.includes('## Further results were found in:');
+			
+			
+			// At least one of these should indicate budget management
+			expect(hasTruncation || hasAdditionalResults).toBeTruthy();
+
+			// Should still contain the footer
+			expect(result).toContain(`Use the "${DOC_FETCH_CONFIG.name}" tool to fetch a document from the library.`);
+
+			// Result should be around our smaller token budget (5k tokens = ~16.5k chars)
+			expect(result.length).toBeLessThan(25000); // Should be controlled by token budget
+		});
 	});
 
 	describe('groupResults', () => {