listpage-next-ai 0.0.272 → 0.0.274

package/dist/cjs/index.d.ts

@@ -18,11 +18,13 @@ export declare interface AgentOptions {
         database?: boolean;
         knowledge?: boolean;
         word?: boolean;
+        documentSearch?: boolean;
         websearch?: boolean;
     };
     databaseOptions?: DatabaseAgentOptions;
     knowledgeOptions?: KnowledgeAgentOptions;
     wordOptions?: WordToolOptions;
+    documentSearchOptions?: DocumentSearchToolOptions;
     websearchOptions?: {};
     model: LanguageModelLike;
     system_prompt: string;
@@ -94,6 +96,10 @@ declare interface DatabaseToolOptions {
     }) => Promise<string>;
 }
 
+declare interface DocumentSearchToolOptions {
+    markdown: string;
+}
+
 declare interface KnowledgeAgentOptions extends KnowledgeToolOptions {
 }
 

package/dist/cjs/index.js

@@ -437,8 +437,189 @@ function createWordTools(options) {
         })
     ];
 }
+class SimpleMarkdownHeaderTextSplitter {
+    constructor(options){
+        this.headersToSplitOn = options.headersToSplitOn;
+    }
+    splitText(text) {
+        const lines = text.split('\n');
+        const docs = [];
+        let currentContent = [];
+        let currentMetadata = {};
+        const headerMap = new Map(this.headersToSplitOn);
+        for (const line of lines){
+            const match = line.match(/^(#{1,6}) (.*)/);
+            if (match) {
+                const hashes = match[1];
+                const title = match[2];
+                if (headerMap.has(hashes)) {
+                    if (currentContent.length > 0 && currentContent.some((l)=>l.trim().length > 0)) {
+                        const contextHeader = this.getHeaderContext(currentMetadata);
+                        docs.push({
+                            pageContent: (contextHeader + currentContent.join('\n')).trim(),
+                            metadata: {
+                                ...currentMetadata
+                            }
+                        });
+                    }
+                    const currentLevel = hashes.length;
+                    const headerName = headerMap.get(hashes);
+                    const newMetadata = {};
+                    for (const [h, name] of this.headersToSplitOn)if (h.length < currentLevel && currentMetadata[name]) newMetadata[name] = currentMetadata[name];
+                    newMetadata[headerName] = title;
+                    currentMetadata = newMetadata;
+                    currentContent = [];
+                    continue;
+                }
+            }
+            currentContent.push(line);
+        }
+        if (currentContent.length > 0 && currentContent.some((l)=>l.trim().length > 0)) {
+            const contextHeader = this.getHeaderContext(currentMetadata);
+            docs.push({
+                pageContent: (contextHeader + currentContent.join('\n')).trim(),
+                metadata: {
+                    ...currentMetadata
+                }
+            });
+        }
+        return docs;
+    }
+    getHeaderContext(metadata) {
+        const levels = [
+            'h1',
+            'h2',
+            'h3',
+            'h4',
+            'h5',
+            'h6'
+        ];
+        const headers = [];
+        for (const level of levels)if (metadata[level]) headers.push(metadata[level]);
+        if (0 === headers.length) return '';
+        return `Start of the section: ${headers.join(' > ')}\n\n`;
+    }
+}
+function simpleTokenize(text) {
+    const regex = /[\u4e00-\u9fa5]|[a-zA-Z0-9]+/g;
+    return text.match(regex) || [];
+}
+class DocumentSearchEngine {
+    constructor(){
+        this.documents = [];
+        this.tokenizedDocs = [];
+        this.bm25 = null;
+    }
+    async loadAndIndex(markdown) {
+        const headersToSplitOn = [
+            [
+                '#',
+                'h1'
+            ],
+            [
+                '##',
+                'h2'
+            ],
+            [
+                '###',
+                'h3'
+            ],
+            [
+                '####',
+                'h4'
+            ]
+        ];
+        const splitter = new SimpleMarkdownHeaderTextSplitter({
+            headersToSplitOn
+        });
+        const splitDocs = splitter.splitText(markdown);
+        const newDocs = splitDocs.map((doc)=>({
+                pageContent: doc.pageContent,
+                metadata: doc.metadata
+            }));
+        const newTokens = newDocs.map((doc)=>{
+            const tokens = simpleTokenize(doc.pageContent);
+            return tokens;
+        });
+        this.documents = [
+            ...this.documents,
+            ...newDocs
+        ];
+        this.tokenizedDocs = [
+            ...this.tokenizedDocs,
+            ...newTokens
+        ];
+        this.bm25 = new BM25(this.tokenizedDocs);
+    }
+    search(query, options = {}) {
+        if (!this.bm25 || 0 === this.documents.length) return [];
+        const { k = 3 } = options;
+        const queryTokens = simpleTokenize(query);
+        const scores = this.documents.map((_, index)=>{
+            const score = this.bm25.getScore(queryTokens, index);
+            return {
+                index,
+                score
+            };
+        });
+        return scores.filter((item)=>item.score > 0).sort((a, b)=>b.score - a.score).slice(0, k).map(({ index, score })=>({
+                document: this.documents[index],
+                score
+            }));
+    }
+    getDocuments() {
+        return this.documents;
+    }
+    clear() {
+        this.documents = [];
+        this.tokenizedDocs = [];
+        this.bm25 = null;
+    }
+}
+function createDocumentSearchTools(options) {
+    const { markdown } = options;
+    const engine = new DocumentSearchEngine();
+    let isIndexed = false;
+    let indexingPromise = null;
+    const ensureIndexed = async ()=>{
+        if (isIndexed) return;
+        if (!indexingPromise) indexingPromise = engine.loadAndIndex(markdown).then(()=>{
+            isIndexed = true;
+            indexingPromise = null;
+        });
+        await indexingPromise;
+    };
+    return [
+        (0, external_langchain_namespaceObject.tool)(async ({ query })=>{
+            try {
+                await ensureIndexed();
+                const results = engine.search(query);
+                if (0 === results.length) return '未找到相关文档内容。';
+                return results.map((res, i)=>{
+                    const metadataStr = Object.entries(res.document.metadata).map(([key, value])=>`${key}: ${value}`).join(', ');
+                    return `[结果 ${i + 1}] (得分: ${res.score.toFixed(4)})\n元数据: { ${metadataStr} }\n内容: ${res.document.pageContent}\n`;
+                }).join('\n---\n');
+            } catch (error) {
+                console.error('Document search failed:', error);
+                return `检索过程中发生错误: ${error instanceof Error ? error.message : String(error)}`;
+            }
+        }, {
+            name: 'search_local_documents',
+            description: '当需要回答关于用户提供文档的问题时使用此工具。该工具会基于 BM25 算法在用户提供的 Markdown 文档中检索相关片段。',
+            schema: {
+                type: 'object',
+                properties: {
+                    query: {
+                        type: 'string',
+                        description: '检索关键词或问题'
+                    }
+                }
+            }
+        })
+    ];
+}
 function createReactAgent(options) {
-    const { name, model, system_prompt, tools = [], middleware = [], features, databaseOptions, knowledgeOptions, wordOptions, inject_current_time, max_iterations = 0, simple = false } = options;
+    const { name, model, system_prompt, tools = [], middleware = [], features, databaseOptions, knowledgeOptions, wordOptions, documentSearchOptions, inject_current_time, max_iterations = 0, simple = false } = options;
     const agent = (0, external_langchain_namespaceObject.createAgent)({
         name,
         model,
@@ -447,6 +628,7 @@ function createReactAgent(options) {
             ...features.database ? createDatabaseTools(databaseOptions) : [],
             ...features.knowledge ? createKnowledgeTools(knowledgeOptions) : [],
             ...features.word ? createWordTools(wordOptions) : [],
+            ...features.documentSearch ? createDocumentSearchTools(documentSearchOptions) : [],
             ...tools
         ].filter(Boolean),
         middleware: [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "listpage-next-ai",
-  "version": "0.0.272",
+  "version": "0.0.274",
   "description": "A React component library for creating filter forms with Ant Design",
   "main": "./dist/cjs/index.js",
   "module": "./dist/esm/index.js",