@llmindset/hf-mcp 0.1.28 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs-search/doc-fetch.d.ts.map +1 -1
- package/dist/docs-search/doc-fetch.js +9 -0
- package/dist/docs-search/doc-fetch.js.map +1 -1
- package/dist/docs-search/docs-semantic-search.js +1 -1
- package/dist/docs-search/docs-semantic-search.js.map +1 -1
- package/package.json +1 -1
- package/src/docs-search/doc-fetch.ts +21 -11
- package/src/docs-search/docs-semantic-search.ts +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"doc-fetch.d.ts","sourceRoot":"","sources":["../../src/docs-search/doc-fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"doc-fetch.d.ts","sourceRoot":"","sources":["../../src/docs-search/doc-fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;CAsBnB,CAAC;AAEX,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,MAAM,CAAC,CAAC;AAErE,qBAAa,YAAY;IACxB,OAAO,CAAC,eAAe,CAAkB;;IAqBzC,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAS1B,KAAK,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBpD,OAAO,CAAC,aAAa;CAoCrB"}
|
|
@@ -30,6 +30,15 @@ export class DocFetchTool {
|
|
|
30
30
|
headingStyle: 'atx',
|
|
31
31
|
codeBlockStyle: 'fenced',
|
|
32
32
|
});
|
|
33
|
+
this.turndownService.remove('head');
|
|
34
|
+
this.turndownService.remove('script');
|
|
35
|
+
this.turndownService.remove((node) => {
|
|
36
|
+
console.log(`${node.nodeName} `);
|
|
37
|
+
if (node.nodeName === 'a' && node.innerHTML.includes('<!-- HTML_TAG_START -->')) {
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
return false;
|
|
41
|
+
});
|
|
33
42
|
}
|
|
34
43
|
validateUrl(hfUrl) {
|
|
35
44
|
if (!hfUrl.startsWith('https://huggingface.co/docs/')) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"doc-fetch.js","sourceRoot":"","sources":["../../src/docs-search/doc-fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC/B,IAAI,EAAE,cAAc;IACpB,WAAW,
|
|
1
|
+
{"version":3,"file":"doc-fetch.js","sourceRoot":"","sources":["../../src/docs-search/doc-fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC/B,IAAI,EAAE,cAAc;IACpB,WAAW,EACV,yHAAyH;IAC1H,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,OAAO,EAAE,CAAC;aACR,MAAM,EAAE;aACR,GAAG,CAAC,EAAE,EAAE,oDAAoD,CAAC;aAC7D,GAAG,CAAC,GAAG,EAAE,gBAAgB,CAAC;aAC1B,QAAQ,CAAC,gCAAgC,CAAC;QAC5C,MAAM,EAAE,CAAC;aACP,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,QAAQ,EAAE;aACV,QAAQ,CAAC,2EAA2E,CAAC;KACvF,CAAC;IACF,WAAW,EAAE;QACZ,KAAK,EAAE,gDAAgD;QACvD,eAAe,EAAE,KAAK;QACtB,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,IAAI;KACnB;CACQ,CAAC;AAIX,MAAM,OAAO,YAAY;IAChB,eAAe,CAAkB;IAEzC;QACC,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,CAAC;YAC1C,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,QAAQ;SACxB,CAAC,CAAC;QACH,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACpC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YACpC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CAAC;gBACjF,OAAO,IAAI,CAAC;YACb,CAAC;YACD,OAAO,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;IACJ,CAAC;IAKD,WAAW,CAAC,KAAa;QACxB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,8BAA8B,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACnE,CAAC;IACF,CAAC;IAKD,KAAK,CAAC,KAAK,CAAC,MAAsB;QACjC,IAAI,CAAC;YACJ,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAEjC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAE7C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,6BAA6B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACxF,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAG1C,MAAM,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAGvE,OAAO,IAAI,CAAC,aAAa,CAAC,mBAAmB,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC1G,CAAC;IACF,CAAC;IAKO,aAAa,CAAC,eAAuB,EAAE,MAAc;QAC5D,MAAM,WAAW,GAAG,cAAc,CAAC,eAAe,CAAC,CAAC;QACpD,MAAM,iBAAiB,GAAG,IAAI,CAAC;QAG/B,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC;QAC1C,MAAM,aAAa,GAAG,UAAU,GAAG,WAAW,CAAC;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;QAGrD,IAAI,SAAS,IAAI,UAAU,EAAE,CAAC;YAC7B,OAAO,iBAAiB,MAAM,mCAAmC,WAAW,iBAAiB,CAAC;QAC/F,CAAC;QAGD,IAAI,WAAW,IAAI,iBAAiB,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACtD,OAAO,eAAe,CAAC;QACxB,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,aAAa,CAAC,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,gBAAgB,EAAE,UAAU,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAGxD,MAAM,UAAU,GAAG,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG,UAAU,GAAG,WAAW,CAAC;QAEzC,IAAI,MAAM,GAAG,KAAK,CAAC;QAGnB,IAAI,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,oCAAoC,gBAAgB,CAAC,IAAI,sBAAsB,UAAU,yBAAyB,CAAC;QAC9H,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;CACD"}
|
|
@@ -4,7 +4,7 @@ import { escapeMarkdown, estimateTokens } from '../utilities.js';
|
|
|
4
4
|
import { DOC_FETCH_CONFIG } from './doc-fetch.js';
|
|
5
5
|
export const DOCS_SEMANTIC_SEARCH_CONFIG = {
|
|
6
6
|
name: 'hf_doc_search',
|
|
7
|
-
description: 'Search
|
|
7
|
+
description: 'Search documentation about all of Hugging Face products and libraries (Transformers, Datasets, Diffusers, Gradio, Hub, and more). Use this for the most up-to-date information ' +
|
|
8
8
|
'Returns excerpts grouped by Product and Document.',
|
|
9
9
|
schema: z.object({
|
|
10
10
|
query: z
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docs-semantic-search.js","sourceRoot":"","sources":["../../src/docs-search/docs-semantic-search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAKlD,MAAM,CAAC,MAAM,2BAA2B,GAAG;IAC1C,IAAI,EAAE,eAAe;IACrB,WAAW,EACV,
|
|
1
|
+
{"version":3,"file":"docs-semantic-search.js","sourceRoot":"","sources":["../../src/docs-search/docs-semantic-search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAKlD,MAAM,CAAC,MAAM,2BAA2B,GAAG;IAC1C,IAAI,EAAE,eAAe;IACrB,WAAW,EACV,iLAAiL;QACjL,mDAAmD;IACpD,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,KAAK,EAAE,CAAC;aACN,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,EAAE,iCAAiC,CAAC;aACzC,GAAG,CAAC,GAAG,EAAE,gBAAgB,CAAC;aAC1B,QAAQ,CAAC,uBAAuB,CAAC;QACnC,OAAO,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CACR,0GAA0G,CAC1G;KACF,CAAC;IACF,WAAW,EAAE;QACZ,KAAK,EAAE,2CAA2C;QAClD,eAAe,EAAE,KAAK;QACtB,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,IAAI;KACnB;CACQ,CAAC;AAmBX,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAKpC,MAAM,OAAO,aAAc,SAAQ,SAAgD;IAC1E,WAAW,CAAS;IAO5B,YAAY,OAAgB,EAAE,MAAM,GAAG,+BAA+B,EAAE,WAAW,GAAG,oBAAoB;QACzG,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IAChC,CAAC;IAMD,KAAK,CAAC,MAAM,CAAC,MAAuB;QACnC,IAAI,CAAC;YACJ,IAAI,CAAC,MAAM,CAAC,KAAK;gBAAE,OAAO,mBAAmB,CAAC;YAE9C,MAAM,SAAS,GAAuB,EAAE,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,SAAS,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YACpC,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAoB,SAAS,CAAC,CAAC;YAEjE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,OAAO,MAAM,CAAC,OAAO;oBACpB,CAAC,CAAC,qCAAqC,MAAM,CAAC,KAAK,iBAAiB,MAAM,CAAC,OAAO,GAAG;oBACrF,CAAC,CAAC,qCAAqC,MAAM,CAAC,KAAK,GAAG,CAAC;YACzD,CAAC;YAED,OAAO,mBAAmB,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACrF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;CACD;AAKD,SAAS,YAAY,CAAC,OAA0B;IAC/C,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0C,CAAC;IAElE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACjD,IAAI,CAAC,YAAY;YAAE,SAAS;QAG5B,MAAM,OAAO,GAAG,MAAM,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,eAAe,CAAC;QAE/E,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,WAAW,EAAE,CAAC;YACjB,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;IACF,CAAC;IAED,OAAO,OAAO,CAAC;AAChB,CAAC;AAKD,SAAS,cAAc,CAAC,WAA8B;IACrD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyC,CAAC;IAEvE,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,aAAa,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,cAAc,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,cAAc,EAAE,CAAC;YACpB,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7B,CAAC;IACF,CAAC;IAED,OAAO,aAAa,CAAC;AACtB,CAAC;AAKD,SAAS,qBAAqB,CAC7B,OAA2B,EAC3B,OAA0B,EAC1B,gBAAyB,EACzB,yBAAkC;IAElC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAGhC,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,OAAO,GACZ,OAAO,CAAC,MAAM,GAAG,CAAC;YACjB,CAAC,CAAC,6BAA6B,cAAc,CAAC,OAAO,CAAC,WAAW;YACjE,CAAC,CAAC,4BAA4B,cAAc,CAAC,OAAO,CAAC,WAAW,CAAC;QAEnE,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACxB,UAAU,IAAI,cAAc,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,SAAS,GAAG,MAAM,CAAC,IAAI;aACzB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;aACvB,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC;aACzB,IAAI,EAAE,CAAC;QAGT,IAAI,gBAAgB,IAAI,SAAS,CAAC,MAAM,GAAG,uBAAuB,IAAI,CAAC,yBAAyB,EAAE,CAAC;YAClG,SAAS;gBACR,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,uBAAuB,CAAC;oBAC/C,oCAAoC,gBAAgB,CAAC,IAAI,yCAAyC,CAAC;YACpG,mBAAmB,GAAG,IAAI,CAAC;QAC5B,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC1B,UAAU,IAAI,cAAc,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IAClD,CAAC;IAGD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;QACxD,KAAK,CAAC,GAAG,EAAE,CAAC;IACb,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAAE,CAAC;AACpE,CAAC;AAKD,SAAS,mBAAmB,CAC3B,KAAa,EACb,OAA0B,EAC1B,aAAsB,EACtB,WAAW,GAAG,oBAAoB;IAElC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,yBAAyB,GAAG,KAAK,CAAC;IAGtC,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,0BAA0B,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACnF,MAAM,MAAM,GAAG,+CAA+C,cAAc,CAAC,KAAK,CAAC,IAAI,UAAU,aAAa,OAAO,CAAC,MAAM,UAAU,CAAC;IACvI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAGnB,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC/D,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa;YAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC5F,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC5F,OAAO,MAAM,GAAG,MAAM,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,MAAM,eAAe,GAA0E,EAAE,CAAC;IAElG,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACtC,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC1C,IAAI,CAAC,YAAY;YAAE,SAAS;QAG5B,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,cAAc,CAAC,WAAW,CAAC,GAAG,WAAW,EAAE,CAAC;YAE/C,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;gBACvC,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC1C,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;oBAAE,SAAS;gBAChC,eAAe,CAAC,IAAI,CAAC;oBACpB,OAAO;oBACP,GAAG;oBACH,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,iBAAiB;oBAClE,KAAK,EAAE,WAAW,CAAC,MAAM;iBACzB,CAAC,CAAC;YACJ,CAAC;YACD,SAAS;QACV,CAAC;QAGD,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACrG,MAAM,aAAa,GAAG,2BAA2B,cAAc,CAAC,OAAO,CAAC,KAAK,gBAAgB,aAAa,CAAC;QAC3G,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAG1B,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAChE,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY;gBAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,YAAY,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAAE,SAAS;YAEhC,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC;YAG9E,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,cAAc,CAAC,WAAW,CAAC,GAAG,WAAW,EAAE,CAAC;gBAC/C,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;gBACpF,SAAS;YACV,CAAC;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,MAAM,UAAU,GAAG,uBAAuB,cAAc,CAAC,SAAS,CAAC,KAAK,GAAG,IAAI,QAAQ,IAAI,CAAC;YAC5F,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAGvB,MAAM,aAAa,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;YAClD,KAAK,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC,IAAI,aAAa,EAAE,CAAC;gBACvD,MAAM,aAAa,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBACvD,MAAM,gBAAgB,GAAG,aAAa,GAAG,WAAW,GAAG,GAAG,CAAC;gBAE3D,MAAM,MAAM,GAAG,qBAAqB,CAAC,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,yBAAyB,CAAC,CAAC;gBAE3G,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;oBACxB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBACxB,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;wBAChC,yBAAyB,GAAG,IAAI,CAAC;oBAClC,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAGD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACpD,KAAK,MAAM,UAAU,IAAI,eAAe,EAAE,CAAC;YAC1C,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7E,KAAK,CAAC,IAAI,CAAC,MAAM,cAAc,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,GAAG,IAAI,OAAO,MAAM,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC;QAC9G,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,KAAK,CAAC,IAAI,CAAC,YAAY,gBAAgB,CAAC,IAAI,8CAA8C,CAAC,CAAC;IAE5F,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC"}
|
package/package.json
CHANGED
|
@@ -4,7 +4,8 @@ import { estimateTokens } from '../utilities.js';
|
|
|
4
4
|
|
|
5
5
|
export const DOC_FETCH_CONFIG = {
|
|
6
6
|
name: 'hf_doc_fetch',
|
|
7
|
-
description:
|
|
7
|
+
description:
|
|
8
|
+
'Fetch a document from the Hugging Face documentation library. For large documents, use offset to get subsequent chunks.',
|
|
8
9
|
schema: z.object({
|
|
9
10
|
doc_url: z
|
|
10
11
|
.string()
|
|
@@ -35,6 +36,15 @@ export class DocFetchTool {
|
|
|
35
36
|
headingStyle: 'atx',
|
|
36
37
|
codeBlockStyle: 'fenced',
|
|
37
38
|
});
|
|
39
|
+
this.turndownService.remove('head');
|
|
40
|
+
this.turndownService.remove('script');
|
|
41
|
+
this.turndownService.remove((node) => {
|
|
42
|
+
console.log(`${node.nodeName} `);
|
|
43
|
+
if (node.nodeName === 'a' && node.innerHTML.includes('<!-- HTML_TAG_START -->')) {
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
return false;
|
|
47
|
+
});
|
|
38
48
|
}
|
|
39
49
|
|
|
40
50
|
/**
|
|
@@ -60,10 +70,10 @@ export class DocFetchTool {
|
|
|
60
70
|
}
|
|
61
71
|
|
|
62
72
|
const htmlContent = await response.text();
|
|
63
|
-
|
|
73
|
+
|
|
64
74
|
// Convert HTML to Markdown
|
|
65
75
|
const fullMarkdownContent = this.turndownService.turndown(htmlContent);
|
|
66
|
-
|
|
76
|
+
|
|
67
77
|
// Apply chunking logic
|
|
68
78
|
return this.applyChunking(fullMarkdownContent, params.offset || 0);
|
|
69
79
|
} catch (error) {
|
|
@@ -77,37 +87,37 @@ export class DocFetchTool {
|
|
|
77
87
|
private applyChunking(markdownContent: string, offset: number): string {
|
|
78
88
|
const totalTokens = estimateTokens(markdownContent);
|
|
79
89
|
const maxTokensPerChunk = 7500;
|
|
80
|
-
|
|
90
|
+
|
|
81
91
|
// Calculate character positions based on tokens
|
|
82
92
|
const totalChars = markdownContent.length;
|
|
83
93
|
const charsPerToken = totalChars / totalTokens;
|
|
84
94
|
const startChar = Math.floor(offset * charsPerToken);
|
|
85
|
-
|
|
95
|
+
|
|
86
96
|
// If offset is beyond document, return error message
|
|
87
97
|
if (startChar >= totalChars) {
|
|
88
98
|
return `Error: Offset ${offset} is beyond the document length (${totalTokens} tokens total).`;
|
|
89
99
|
}
|
|
90
|
-
|
|
100
|
+
|
|
91
101
|
// If document is small enough and no offset, return as-is
|
|
92
102
|
if (totalTokens <= maxTokensPerChunk && offset === 0) {
|
|
93
103
|
return markdownContent;
|
|
94
104
|
}
|
|
95
|
-
|
|
105
|
+
|
|
96
106
|
const maxCharsPerChunk = Math.floor(maxTokensPerChunk * charsPerToken);
|
|
97
107
|
const endChar = Math.min(startChar + maxCharsPerChunk, totalChars);
|
|
98
108
|
const chunk = markdownContent.slice(startChar, endChar);
|
|
99
|
-
|
|
109
|
+
|
|
100
110
|
// Calculate next offset
|
|
101
111
|
const nextOffset = offset + estimateTokens(chunk);
|
|
102
112
|
const hasMore = nextOffset < totalTokens;
|
|
103
|
-
|
|
113
|
+
|
|
104
114
|
let result = chunk;
|
|
105
|
-
|
|
115
|
+
|
|
106
116
|
// Add truncation message if there's more content
|
|
107
117
|
if (hasMore) {
|
|
108
118
|
result += `\n\n=== DOCUMENT TRUNCATED. CALL ${DOC_FETCH_CONFIG.name} WITH AN OFFSET OF ${nextOffset} FOR THE NEXT CHUNK ===`;
|
|
109
119
|
}
|
|
110
|
-
|
|
120
|
+
|
|
111
121
|
return result;
|
|
112
122
|
}
|
|
113
123
|
}
|
|
@@ -9,7 +9,7 @@ import { DOC_FETCH_CONFIG } from './doc-fetch.js';
|
|
|
9
9
|
export const DOCS_SEMANTIC_SEARCH_CONFIG = {
|
|
10
10
|
name: 'hf_doc_search',
|
|
11
11
|
description:
|
|
12
|
-
'Search
|
|
12
|
+
'Search documentation about all of Hugging Face products and libraries (Transformers, Datasets, Diffusers, Gradio, Hub, and more). Use this for the most up-to-date information ' +
|
|
13
13
|
'Returns excerpts grouped by Product and Document.',
|
|
14
14
|
schema: z.object({
|
|
15
15
|
query: z
|