@mastra/rag 0.1.19-alpha.3 → 0.1.19-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +8 -0
- package/dist/_tsup-dts-rollup.d.cts +237 -5
- package/dist/_tsup-dts-rollup.d.ts +237 -5
- package/dist/index.cjs +3980 -14
- package/dist/index.js +3977 -11
- package/package.json +2 -4
- package/src/document/document.test.ts +123 -2
- package/src/document/document.ts +15 -21
- package/src/document/extractors/index.ts +5 -0
- package/src/document/extractors/keywords.test.ts +119 -0
- package/src/document/extractors/keywords.ts +123 -0
- package/src/document/extractors/questions.test.ts +120 -0
- package/src/document/extractors/questions.ts +126 -0
- package/src/document/extractors/summary.test.ts +107 -0
- package/src/document/extractors/summary.ts +130 -0
- package/src/document/extractors/title.test.ts +121 -0
- package/src/document/extractors/title.ts +210 -0
- package/src/document/extractors/types.ts +40 -0
- package/src/document/types.ts +5 -33
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
|
|
2
|
-
> @mastra/rag@0.1.19-alpha.
|
|
2
|
+
> @mastra/rag@0.1.19-alpha.4 build /home/runner/work/mastra/mastra/packages/rag
|
|
3
3
|
> tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
|
|
4
4
|
|
|
5
5
|
[34mCLI[39m Building entry: src/index.ts
|
|
6
6
|
[34mCLI[39m Using tsconfig: tsconfig.json
|
|
7
7
|
[34mCLI[39m tsup v8.4.0
|
|
8
8
|
[34mTSC[39m Build start
|
|
9
|
-
[32mTSC[39m ⚡️ Build success in
|
|
9
|
+
[32mTSC[39m ⚡️ Build success in 16612ms
|
|
10
10
|
[34mDTS[39m Build start
|
|
11
11
|
[34mCLI[39m Target: es2022
|
|
12
12
|
Analysis will use the bundled TypeScript version 5.8.2
|
|
13
13
|
[36mWriting package typings: /home/runner/work/mastra/mastra/packages/rag/dist/_tsup-dts-rollup.d.ts[39m
|
|
14
14
|
Analysis will use the bundled TypeScript version 5.8.2
|
|
15
15
|
[36mWriting package typings: /home/runner/work/mastra/mastra/packages/rag/dist/_tsup-dts-rollup.d.cts[39m
|
|
16
|
-
[32mDTS[39m ⚡️ Build success in
|
|
16
|
+
[32mDTS[39m ⚡️ Build success in 15610ms
|
|
17
17
|
[34mCLI[39m Cleaning output folder
|
|
18
18
|
[34mESM[39m Build start
|
|
19
19
|
[34mCJS[39m Build start
|
|
20
|
-
[
|
|
21
|
-
[
|
|
22
|
-
[
|
|
23
|
-
[
|
|
20
|
+
[32mESM[39m [1mdist/index.js [22m[32m218.60 KB[39m
|
|
21
|
+
[32mESM[39m ⚡️ Build success in 4018ms
|
|
22
|
+
[32mCJS[39m [1mdist/index.cjs [22m[32m220.36 KB[39m
|
|
23
|
+
[32mCJS[39m ⚡️ Build success in 4018ms
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# @mastra/rag
|
|
2
2
|
|
|
3
|
+
## 0.1.19-alpha.4
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- f850f80: use custom extractors rather than llamaindex versions, and remove @llamaindex/core and @llamaindex/env packages
|
|
8
|
+
- Updated dependencies [7e92011]
|
|
9
|
+
- @mastra/core@0.9.0-alpha.4
|
|
10
|
+
|
|
3
11
|
## 0.1.19-alpha.3
|
|
4
12
|
|
|
5
13
|
### Patch Changes
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import { BaseExtractor } from 'llamaindex';
|
|
2
|
+
import type { BaseNode } from 'llamaindex';
|
|
1
3
|
import { createTool } from '@mastra/core/tools';
|
|
2
4
|
import { Document } from 'llamaindex';
|
|
3
5
|
import type { EmbeddingModel } from 'ai';
|
|
4
6
|
import type { KeywordExtractPrompt } from 'llamaindex';
|
|
5
|
-
import type { LLM } from 'llamaindex';
|
|
6
7
|
import type { MastraLanguageModel } from '@mastra/core/agent';
|
|
7
8
|
import type { MastraVector } from '@mastra/core/vector';
|
|
8
9
|
import type { QueryResult } from '@mastra/core/vector';
|
|
@@ -22,6 +23,10 @@ declare const ASTRA_PROMPT = "When querying Astra, you can ONLY use the operator
|
|
|
22
23
|
export { ASTRA_PROMPT }
|
|
23
24
|
export { ASTRA_PROMPT as ASTRA_PROMPT_alias_1 }
|
|
24
25
|
|
|
26
|
+
declare const baseLLM: MastraLanguageModel;
|
|
27
|
+
export { baseLLM }
|
|
28
|
+
export { baseLLM as baseLLM_alias_1 }
|
|
29
|
+
|
|
25
30
|
export declare class CharacterTransformer extends TextTransformer {
|
|
26
31
|
protected separator: string;
|
|
27
32
|
protected isSeparatorRegex: boolean;
|
|
@@ -134,6 +139,13 @@ export { defaultVectorQueryDescription }
|
|
|
134
139
|
export { defaultVectorQueryDescription as defaultVectorQueryDescription_alias_1 }
|
|
135
140
|
export { defaultVectorQueryDescription as defaultVectorQueryDescription_alias_2 }
|
|
136
141
|
|
|
142
|
+
declare type ExtractKeyword = {
|
|
143
|
+
/**
|
|
144
|
+
* Comma-separated keywords extracted from the node. May be empty if extraction fails.
|
|
145
|
+
*/
|
|
146
|
+
excerptKeywords: string;
|
|
147
|
+
};
|
|
148
|
+
|
|
137
149
|
declare type ExtractParams = {
|
|
138
150
|
title?: TitleExtractorsArgs | boolean;
|
|
139
151
|
summary?: SummaryExtractArgs | boolean;
|
|
@@ -143,6 +155,23 @@ declare type ExtractParams = {
|
|
|
143
155
|
export { ExtractParams }
|
|
144
156
|
export { ExtractParams as ExtractParams_alias_1 }
|
|
145
157
|
|
|
158
|
+
declare type ExtractQuestion = {
|
|
159
|
+
/**
|
|
160
|
+
* Questions extracted from the node as a string (may be empty if extraction fails).
|
|
161
|
+
*/
|
|
162
|
+
questionsThisExcerptCanAnswer: string;
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
declare type ExtractSummary = {
|
|
166
|
+
sectionSummary?: string;
|
|
167
|
+
prevSectionSummary?: string;
|
|
168
|
+
nextSectionSummary?: string;
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
declare type ExtractTitle = {
|
|
172
|
+
documentTitle: string;
|
|
173
|
+
};
|
|
174
|
+
|
|
146
175
|
declare const filterDescription = "JSON-formatted criteria to refine search results.\n- ALWAYS provide a filter value\n- If no filter is provided, use the default (\"{}\")\n- MUST be a valid, complete JSON object with proper quotes and brackets\n- Uses provided filter if specified\n- Default: \"{}\" (no filtering)\n- Example for no filtering: \"filter\": \"{}\"\n- Example: '{\"category\": \"health\"}'\n- Based on query intent\n- Do NOT use single quotes or unquoted properties\n- IMPORTANT: Always ensure JSON is properly closed with matching brackets\n- Multiple filters can be combined";
|
|
147
176
|
export { filterDescription }
|
|
148
177
|
export { filterDescription as filterDescription_alias_1 }
|
|
@@ -226,13 +255,66 @@ export declare class HTMLSectionTransformer {
|
|
|
226
255
|
}
|
|
227
256
|
|
|
228
257
|
declare type KeywordExtractArgs = {
|
|
229
|
-
llm?:
|
|
258
|
+
llm?: MastraLanguageModel;
|
|
230
259
|
keywords?: number;
|
|
231
260
|
promptTemplate?: KeywordExtractPrompt['template'];
|
|
232
261
|
};
|
|
233
262
|
export { KeywordExtractArgs }
|
|
234
263
|
export { KeywordExtractArgs as KeywordExtractArgs_alias_1 }
|
|
235
264
|
|
|
265
|
+
/**
|
|
266
|
+
* Extract keywords from a list of nodes.
|
|
267
|
+
*/
|
|
268
|
+
declare class KeywordExtractor extends BaseExtractor {
|
|
269
|
+
/**
|
|
270
|
+
* MastraLanguageModel instance.
|
|
271
|
+
* @type {MastraLanguageModel}
|
|
272
|
+
*/
|
|
273
|
+
llm: MastraLanguageModel;
|
|
274
|
+
/**
|
|
275
|
+
* Number of keywords to extract.
|
|
276
|
+
* @type {number}
|
|
277
|
+
* @default 5
|
|
278
|
+
*/
|
|
279
|
+
keywords: number;
|
|
280
|
+
/**
|
|
281
|
+
* The prompt template to use for the question extractor.
|
|
282
|
+
* @type {string}
|
|
283
|
+
*/
|
|
284
|
+
promptTemplate: KeywordExtractPrompt;
|
|
285
|
+
/**
|
|
286
|
+
* Constructor for the KeywordExtractor class.
|
|
287
|
+
* @param {MastraLanguageModel} llm MastraLanguageModel instance.
|
|
288
|
+
* @param {number} keywords Number of keywords to extract.
|
|
289
|
+
* @param {string} [promptTemplate] Optional custom prompt template (must include {context})
|
|
290
|
+
* @throws {Error} If keywords is less than 1.
|
|
291
|
+
*/
|
|
292
|
+
constructor(options?: KeywordExtractArgs);
|
|
293
|
+
/**
|
|
294
|
+
*
|
|
295
|
+
* @param node Node to extract keywords from.
|
|
296
|
+
* @returns Keywords extracted from the node.
|
|
297
|
+
*/
|
|
298
|
+
/**
|
|
299
|
+
* Extract keywords from a node. Returns an object with a comma-separated string of keywords, or an empty string if extraction fails.
|
|
300
|
+
* Adds error handling for malformed/empty LLM output.
|
|
301
|
+
*/
|
|
302
|
+
extractKeywordsFromNodes(node: BaseNode): Promise<ExtractKeyword>;
|
|
303
|
+
/**
|
|
304
|
+
*
|
|
305
|
+
* @param nodes Nodes to extract keywords from.
|
|
306
|
+
* @returns Keywords extracted from the nodes.
|
|
307
|
+
*/
|
|
308
|
+
/**
|
|
309
|
+
* Extract keywords from an array of nodes. Always returns an array (may be empty).
|
|
310
|
+
* @param nodes Nodes to extract keywords from.
|
|
311
|
+
* @returns Array of keyword extraction results.
|
|
312
|
+
*/
|
|
313
|
+
extract(nodes: BaseNode[]): Promise<Array<ExtractKeyword>>;
|
|
314
|
+
}
|
|
315
|
+
export { KeywordExtractor }
|
|
316
|
+
export { KeywordExtractor as KeywordExtractor_alias_1 }
|
|
317
|
+
|
|
236
318
|
declare enum Language {
|
|
237
319
|
CPP = "cpp",
|
|
238
320
|
GO = "go",
|
|
@@ -354,7 +436,7 @@ export { queryTextDescription as queryTextDescription_alias_1 }
|
|
|
354
436
|
export { queryTextDescription as queryTextDescription_alias_2 }
|
|
355
437
|
|
|
356
438
|
declare type QuestionAnswerExtractArgs = {
|
|
357
|
-
llm?:
|
|
439
|
+
llm?: MastraLanguageModel;
|
|
358
440
|
questions?: number;
|
|
359
441
|
promptTemplate?: QuestionExtractPrompt['template'];
|
|
360
442
|
embeddingOnly?: boolean;
|
|
@@ -362,6 +444,56 @@ declare type QuestionAnswerExtractArgs = {
|
|
|
362
444
|
export { QuestionAnswerExtractArgs }
|
|
363
445
|
export { QuestionAnswerExtractArgs as QuestionAnswerExtractArgs_alias_1 }
|
|
364
446
|
|
|
447
|
+
/**
|
|
448
|
+
* Extract questions from a list of nodes.
|
|
449
|
+
*/
|
|
450
|
+
declare class QuestionsAnsweredExtractor extends BaseExtractor {
|
|
451
|
+
/**
|
|
452
|
+
* MastraLanguageModel instance.
|
|
453
|
+
* @type {MastraLanguageModel}
|
|
454
|
+
*/
|
|
455
|
+
llm: MastraLanguageModel;
|
|
456
|
+
/**
|
|
457
|
+
* Number of questions to generate.
|
|
458
|
+
* @type {number}
|
|
459
|
+
* @default 5
|
|
460
|
+
*/
|
|
461
|
+
questions: number;
|
|
462
|
+
/**
|
|
463
|
+
* The prompt template to use for the question extractor.
|
|
464
|
+
* @type {string}
|
|
465
|
+
*/
|
|
466
|
+
promptTemplate: QuestionExtractPrompt;
|
|
467
|
+
/**
|
|
468
|
+
* Wheter to use metadata for embeddings only
|
|
469
|
+
* @type {boolean}
|
|
470
|
+
* @default false
|
|
471
|
+
*/
|
|
472
|
+
embeddingOnly: boolean;
|
|
473
|
+
/**
|
|
474
|
+
* Constructor for the QuestionsAnsweredExtractor class.
|
|
475
|
+
* @param {MastraLanguageModel} llm MastraLanguageModel instance.
|
|
476
|
+
* @param {number} questions Number of questions to generate.
|
|
477
|
+
* @param {QuestionExtractPrompt['template']} promptTemplate Optional custom prompt template (should include {context}).
|
|
478
|
+
* @param {boolean} embeddingOnly Whether to use metadata for embeddings only.
|
|
479
|
+
*/
|
|
480
|
+
constructor(options?: QuestionAnswerExtractArgs);
|
|
481
|
+
/**
|
|
482
|
+
* Extract answered questions from a node.
|
|
483
|
+
* @param {BaseNode} node Node to extract questions from.
|
|
484
|
+
* @returns {Promise<Array<ExtractQuestion> | Array<{}>>} Questions extracted from the node.
|
|
485
|
+
*/
|
|
486
|
+
extractQuestionsFromNode(node: BaseNode): Promise<ExtractQuestion>;
|
|
487
|
+
/**
|
|
488
|
+
* Extract answered questions from a list of nodes.
|
|
489
|
+
* @param {BaseNode[]} nodes Nodes to extract questions from.
|
|
490
|
+
* @returns {Promise<Array<ExtractQuestion> | Array<{}>>} Questions extracted from the nodes.
|
|
491
|
+
*/
|
|
492
|
+
extract(nodes: BaseNode[]): Promise<Array<ExtractQuestion> | Array<object>>;
|
|
493
|
+
}
|
|
494
|
+
export { QuestionsAnsweredExtractor }
|
|
495
|
+
export { QuestionsAnsweredExtractor as QuestionsAnsweredExtractor_alias_1 }
|
|
496
|
+
|
|
365
497
|
declare interface RankedNode extends GraphNode {
|
|
366
498
|
score: number;
|
|
367
499
|
}
|
|
@@ -528,14 +660,61 @@ export declare function splitTextOnTokens({ text, tokenizer }: {
|
|
|
528
660
|
tokenizer: Tokenizer;
|
|
529
661
|
}): string[];
|
|
530
662
|
|
|
663
|
+
declare const STRIP_REGEX: RegExp;
|
|
664
|
+
export { STRIP_REGEX }
|
|
665
|
+
export { STRIP_REGEX as STRIP_REGEX_alias_1 }
|
|
666
|
+
|
|
531
667
|
declare type SummaryExtractArgs = {
|
|
532
|
-
llm?:
|
|
668
|
+
llm?: MastraLanguageModel;
|
|
533
669
|
summaries?: string[];
|
|
534
670
|
promptTemplate?: SummaryPrompt['template'];
|
|
535
671
|
};
|
|
536
672
|
export { SummaryExtractArgs }
|
|
537
673
|
export { SummaryExtractArgs as SummaryExtractArgs_alias_1 }
|
|
538
674
|
|
|
675
|
+
/**
|
|
676
|
+
* Summarize an array of nodes using a custom LLM.
|
|
677
|
+
*
|
|
678
|
+
* @param nodes Array of node-like objects
|
|
679
|
+
* @param options Summary extraction options
|
|
680
|
+
* @returns Array of summary results
|
|
681
|
+
*/
|
|
682
|
+
declare class SummaryExtractor extends BaseExtractor {
|
|
683
|
+
/**
|
|
684
|
+
* MastraLanguageModel instance.
|
|
685
|
+
* @type {MastraLanguageModel}
|
|
686
|
+
*/
|
|
687
|
+
private llm;
|
|
688
|
+
/**
|
|
689
|
+
* List of summaries to extract: 'self', 'prev', 'next'
|
|
690
|
+
* @type {string[]}
|
|
691
|
+
*/
|
|
692
|
+
summaries: string[];
|
|
693
|
+
/**
|
|
694
|
+
* The prompt template to use for the summary extractor.
|
|
695
|
+
* @type {string}
|
|
696
|
+
*/
|
|
697
|
+
promptTemplate: SummaryPrompt;
|
|
698
|
+
private selfSummary;
|
|
699
|
+
private prevSummary;
|
|
700
|
+
private nextSummary;
|
|
701
|
+
constructor(options?: SummaryExtractArgs);
|
|
702
|
+
/**
|
|
703
|
+
* Extract summary from a node.
|
|
704
|
+
* @param {BaseNode} node Node to extract summary from.
|
|
705
|
+
* @returns {Promise<string>} Summary extracted from the node.
|
|
706
|
+
*/
|
|
707
|
+
generateNodeSummary(node: BaseNode): Promise<string>;
|
|
708
|
+
/**
|
|
709
|
+
* Extract summaries from a list of nodes.
|
|
710
|
+
* @param {BaseNode[]} nodes Nodes to extract summaries from.
|
|
711
|
+
* @returns {Promise<ExtractSummary[]>} Summaries extracted from the nodes.
|
|
712
|
+
*/
|
|
713
|
+
extract(nodes: BaseNode[]): Promise<ExtractSummary[]>;
|
|
714
|
+
}
|
|
715
|
+
export { SummaryExtractor }
|
|
716
|
+
export { SummaryExtractor as SummaryExtractor_alias_1 }
|
|
717
|
+
|
|
539
718
|
/**
|
|
540
719
|
* TODO: GraphRAG Enhancements
|
|
541
720
|
* - Add support for more edge types (sequential, hierarchical, citation, etc)
|
|
@@ -564,8 +743,61 @@ export declare abstract class TextTransformer implements Transformer {
|
|
|
564
743
|
protected mergeSplits(splits: string[], separator: string): string[];
|
|
565
744
|
}
|
|
566
745
|
|
|
746
|
+
/**
|
|
747
|
+
* Extract title from a list of nodes.
|
|
748
|
+
*/
|
|
749
|
+
declare class TitleExtractor extends BaseExtractor {
|
|
750
|
+
/**
|
|
751
|
+
* MastraLanguageModel instance.
|
|
752
|
+
* @type {MastraLanguageModel}
|
|
753
|
+
*/
|
|
754
|
+
llm: MastraLanguageModel;
|
|
755
|
+
/**
|
|
756
|
+
* Can work for mixture of text and non-text nodes
|
|
757
|
+
* @type {boolean}
|
|
758
|
+
* @default false
|
|
759
|
+
*/
|
|
760
|
+
isTextNodeOnly: boolean;
|
|
761
|
+
/**
|
|
762
|
+
* Number of nodes to extrct titles from.
|
|
763
|
+
* @type {number}
|
|
764
|
+
* @default 5
|
|
765
|
+
*/
|
|
766
|
+
nodes: number;
|
|
767
|
+
/**
|
|
768
|
+
* The prompt template to use for the title extractor.
|
|
769
|
+
* @type {string}
|
|
770
|
+
*/
|
|
771
|
+
nodeTemplate: TitleExtractorPrompt;
|
|
772
|
+
/**
|
|
773
|
+
* The prompt template to merge title with..
|
|
774
|
+
* @type {string}
|
|
775
|
+
*/
|
|
776
|
+
combineTemplate: TitleCombinePrompt;
|
|
777
|
+
/**
|
|
778
|
+
* Constructor for the TitleExtractor class.
|
|
779
|
+
* @param {MastraLanguageModel} llm MastraLanguageModel instance.
|
|
780
|
+
* @param {number} nodes Number of nodes to extract titles from.
|
|
781
|
+
* @param {TitleExtractorPrompt} nodeTemplate The prompt template to use for the title extractor.
|
|
782
|
+
* @param {string} combineTemplate The prompt template to merge title with..
|
|
783
|
+
*/
|
|
784
|
+
constructor(options?: TitleExtractorsArgs);
|
|
785
|
+
/**
|
|
786
|
+
* Extract titles from a list of nodes.
|
|
787
|
+
* @param {BaseNode[]} nodes Nodes to extract titles from.
|
|
788
|
+
* @returns {Promise<BaseNode<ExtractTitle>[]>} Titles extracted from the nodes.
|
|
789
|
+
*/
|
|
790
|
+
extract(nodes: BaseNode[]): Promise<Array<ExtractTitle>>;
|
|
791
|
+
private filterNodes;
|
|
792
|
+
private separateNodesByDocument;
|
|
793
|
+
private extractTitles;
|
|
794
|
+
private getTitlesCandidates;
|
|
795
|
+
}
|
|
796
|
+
export { TitleExtractor }
|
|
797
|
+
export { TitleExtractor as TitleExtractor_alias_1 }
|
|
798
|
+
|
|
567
799
|
declare type TitleExtractorsArgs = {
|
|
568
|
-
llm?:
|
|
800
|
+
llm?: MastraLanguageModel;
|
|
569
801
|
nodes?: number;
|
|
570
802
|
nodeTemplate?: TitleExtractorPrompt['template'];
|
|
571
803
|
combineTemplate?: TitleCombinePrompt['template'];
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import { BaseExtractor } from 'llamaindex';
|
|
2
|
+
import type { BaseNode } from 'llamaindex';
|
|
1
3
|
import { createTool } from '@mastra/core/tools';
|
|
2
4
|
import { Document } from 'llamaindex';
|
|
3
5
|
import type { EmbeddingModel } from 'ai';
|
|
4
6
|
import type { KeywordExtractPrompt } from 'llamaindex';
|
|
5
|
-
import type { LLM } from 'llamaindex';
|
|
6
7
|
import type { MastraLanguageModel } from '@mastra/core/agent';
|
|
7
8
|
import type { MastraVector } from '@mastra/core/vector';
|
|
8
9
|
import type { QueryResult } from '@mastra/core/vector';
|
|
@@ -22,6 +23,10 @@ declare const ASTRA_PROMPT = "When querying Astra, you can ONLY use the operator
|
|
|
22
23
|
export { ASTRA_PROMPT }
|
|
23
24
|
export { ASTRA_PROMPT as ASTRA_PROMPT_alias_1 }
|
|
24
25
|
|
|
26
|
+
declare const baseLLM: MastraLanguageModel;
|
|
27
|
+
export { baseLLM }
|
|
28
|
+
export { baseLLM as baseLLM_alias_1 }
|
|
29
|
+
|
|
25
30
|
export declare class CharacterTransformer extends TextTransformer {
|
|
26
31
|
protected separator: string;
|
|
27
32
|
protected isSeparatorRegex: boolean;
|
|
@@ -134,6 +139,13 @@ export { defaultVectorQueryDescription }
|
|
|
134
139
|
export { defaultVectorQueryDescription as defaultVectorQueryDescription_alias_1 }
|
|
135
140
|
export { defaultVectorQueryDescription as defaultVectorQueryDescription_alias_2 }
|
|
136
141
|
|
|
142
|
+
declare type ExtractKeyword = {
|
|
143
|
+
/**
|
|
144
|
+
* Comma-separated keywords extracted from the node. May be empty if extraction fails.
|
|
145
|
+
*/
|
|
146
|
+
excerptKeywords: string;
|
|
147
|
+
};
|
|
148
|
+
|
|
137
149
|
declare type ExtractParams = {
|
|
138
150
|
title?: TitleExtractorsArgs | boolean;
|
|
139
151
|
summary?: SummaryExtractArgs | boolean;
|
|
@@ -143,6 +155,23 @@ declare type ExtractParams = {
|
|
|
143
155
|
export { ExtractParams }
|
|
144
156
|
export { ExtractParams as ExtractParams_alias_1 }
|
|
145
157
|
|
|
158
|
+
declare type ExtractQuestion = {
|
|
159
|
+
/**
|
|
160
|
+
* Questions extracted from the node as a string (may be empty if extraction fails).
|
|
161
|
+
*/
|
|
162
|
+
questionsThisExcerptCanAnswer: string;
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
declare type ExtractSummary = {
|
|
166
|
+
sectionSummary?: string;
|
|
167
|
+
prevSectionSummary?: string;
|
|
168
|
+
nextSectionSummary?: string;
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
declare type ExtractTitle = {
|
|
172
|
+
documentTitle: string;
|
|
173
|
+
};
|
|
174
|
+
|
|
146
175
|
declare const filterDescription = "JSON-formatted criteria to refine search results.\n- ALWAYS provide a filter value\n- If no filter is provided, use the default (\"{}\")\n- MUST be a valid, complete JSON object with proper quotes and brackets\n- Uses provided filter if specified\n- Default: \"{}\" (no filtering)\n- Example for no filtering: \"filter\": \"{}\"\n- Example: '{\"category\": \"health\"}'\n- Based on query intent\n- Do NOT use single quotes or unquoted properties\n- IMPORTANT: Always ensure JSON is properly closed with matching brackets\n- Multiple filters can be combined";
|
|
147
176
|
export { filterDescription }
|
|
148
177
|
export { filterDescription as filterDescription_alias_1 }
|
|
@@ -226,13 +255,66 @@ export declare class HTMLSectionTransformer {
|
|
|
226
255
|
}
|
|
227
256
|
|
|
228
257
|
declare type KeywordExtractArgs = {
|
|
229
|
-
llm?:
|
|
258
|
+
llm?: MastraLanguageModel;
|
|
230
259
|
keywords?: number;
|
|
231
260
|
promptTemplate?: KeywordExtractPrompt['template'];
|
|
232
261
|
};
|
|
233
262
|
export { KeywordExtractArgs }
|
|
234
263
|
export { KeywordExtractArgs as KeywordExtractArgs_alias_1 }
|
|
235
264
|
|
|
265
|
+
/**
|
|
266
|
+
* Extract keywords from a list of nodes.
|
|
267
|
+
*/
|
|
268
|
+
declare class KeywordExtractor extends BaseExtractor {
|
|
269
|
+
/**
|
|
270
|
+
* MastraLanguageModel instance.
|
|
271
|
+
* @type {MastraLanguageModel}
|
|
272
|
+
*/
|
|
273
|
+
llm: MastraLanguageModel;
|
|
274
|
+
/**
|
|
275
|
+
* Number of keywords to extract.
|
|
276
|
+
* @type {number}
|
|
277
|
+
* @default 5
|
|
278
|
+
*/
|
|
279
|
+
keywords: number;
|
|
280
|
+
/**
|
|
281
|
+
* The prompt template to use for the question extractor.
|
|
282
|
+
* @type {string}
|
|
283
|
+
*/
|
|
284
|
+
promptTemplate: KeywordExtractPrompt;
|
|
285
|
+
/**
|
|
286
|
+
* Constructor for the KeywordExtractor class.
|
|
287
|
+
* @param {MastraLanguageModel} llm MastraLanguageModel instance.
|
|
288
|
+
* @param {number} keywords Number of keywords to extract.
|
|
289
|
+
* @param {string} [promptTemplate] Optional custom prompt template (must include {context})
|
|
290
|
+
* @throws {Error} If keywords is less than 1.
|
|
291
|
+
*/
|
|
292
|
+
constructor(options?: KeywordExtractArgs);
|
|
293
|
+
/**
|
|
294
|
+
*
|
|
295
|
+
* @param node Node to extract keywords from.
|
|
296
|
+
* @returns Keywords extracted from the node.
|
|
297
|
+
*/
|
|
298
|
+
/**
|
|
299
|
+
* Extract keywords from a node. Returns an object with a comma-separated string of keywords, or an empty string if extraction fails.
|
|
300
|
+
* Adds error handling for malformed/empty LLM output.
|
|
301
|
+
*/
|
|
302
|
+
extractKeywordsFromNodes(node: BaseNode): Promise<ExtractKeyword>;
|
|
303
|
+
/**
|
|
304
|
+
*
|
|
305
|
+
* @param nodes Nodes to extract keywords from.
|
|
306
|
+
* @returns Keywords extracted from the nodes.
|
|
307
|
+
*/
|
|
308
|
+
/**
|
|
309
|
+
* Extract keywords from an array of nodes. Always returns an array (may be empty).
|
|
310
|
+
* @param nodes Nodes to extract keywords from.
|
|
311
|
+
* @returns Array of keyword extraction results.
|
|
312
|
+
*/
|
|
313
|
+
extract(nodes: BaseNode[]): Promise<Array<ExtractKeyword>>;
|
|
314
|
+
}
|
|
315
|
+
export { KeywordExtractor }
|
|
316
|
+
export { KeywordExtractor as KeywordExtractor_alias_1 }
|
|
317
|
+
|
|
236
318
|
declare enum Language {
|
|
237
319
|
CPP = "cpp",
|
|
238
320
|
GO = "go",
|
|
@@ -354,7 +436,7 @@ export { queryTextDescription as queryTextDescription_alias_1 }
|
|
|
354
436
|
export { queryTextDescription as queryTextDescription_alias_2 }
|
|
355
437
|
|
|
356
438
|
declare type QuestionAnswerExtractArgs = {
|
|
357
|
-
llm?:
|
|
439
|
+
llm?: MastraLanguageModel;
|
|
358
440
|
questions?: number;
|
|
359
441
|
promptTemplate?: QuestionExtractPrompt['template'];
|
|
360
442
|
embeddingOnly?: boolean;
|
|
@@ -362,6 +444,56 @@ declare type QuestionAnswerExtractArgs = {
|
|
|
362
444
|
export { QuestionAnswerExtractArgs }
|
|
363
445
|
export { QuestionAnswerExtractArgs as QuestionAnswerExtractArgs_alias_1 }
|
|
364
446
|
|
|
447
|
+
/**
|
|
448
|
+
* Extract questions from a list of nodes.
|
|
449
|
+
*/
|
|
450
|
+
declare class QuestionsAnsweredExtractor extends BaseExtractor {
|
|
451
|
+
/**
|
|
452
|
+
* MastraLanguageModel instance.
|
|
453
|
+
* @type {MastraLanguageModel}
|
|
454
|
+
*/
|
|
455
|
+
llm: MastraLanguageModel;
|
|
456
|
+
/**
|
|
457
|
+
* Number of questions to generate.
|
|
458
|
+
* @type {number}
|
|
459
|
+
* @default 5
|
|
460
|
+
*/
|
|
461
|
+
questions: number;
|
|
462
|
+
/**
|
|
463
|
+
* The prompt template to use for the question extractor.
|
|
464
|
+
* @type {string}
|
|
465
|
+
*/
|
|
466
|
+
promptTemplate: QuestionExtractPrompt;
|
|
467
|
+
/**
|
|
468
|
+
* Wheter to use metadata for embeddings only
|
|
469
|
+
* @type {boolean}
|
|
470
|
+
* @default false
|
|
471
|
+
*/
|
|
472
|
+
embeddingOnly: boolean;
|
|
473
|
+
/**
|
|
474
|
+
* Constructor for the QuestionsAnsweredExtractor class.
|
|
475
|
+
* @param {MastraLanguageModel} llm MastraLanguageModel instance.
|
|
476
|
+
* @param {number} questions Number of questions to generate.
|
|
477
|
+
* @param {QuestionExtractPrompt['template']} promptTemplate Optional custom prompt template (should include {context}).
|
|
478
|
+
* @param {boolean} embeddingOnly Whether to use metadata for embeddings only.
|
|
479
|
+
*/
|
|
480
|
+
constructor(options?: QuestionAnswerExtractArgs);
|
|
481
|
+
/**
|
|
482
|
+
* Extract answered questions from a node.
|
|
483
|
+
* @param {BaseNode} node Node to extract questions from.
|
|
484
|
+
* @returns {Promise<Array<ExtractQuestion> | Array<{}>>} Questions extracted from the node.
|
|
485
|
+
*/
|
|
486
|
+
extractQuestionsFromNode(node: BaseNode): Promise<ExtractQuestion>;
|
|
487
|
+
/**
|
|
488
|
+
* Extract answered questions from a list of nodes.
|
|
489
|
+
* @param {BaseNode[]} nodes Nodes to extract questions from.
|
|
490
|
+
* @returns {Promise<Array<ExtractQuestion> | Array<{}>>} Questions extracted from the nodes.
|
|
491
|
+
*/
|
|
492
|
+
extract(nodes: BaseNode[]): Promise<Array<ExtractQuestion> | Array<object>>;
|
|
493
|
+
}
|
|
494
|
+
export { QuestionsAnsweredExtractor }
|
|
495
|
+
export { QuestionsAnsweredExtractor as QuestionsAnsweredExtractor_alias_1 }
|
|
496
|
+
|
|
365
497
|
declare interface RankedNode extends GraphNode {
|
|
366
498
|
score: number;
|
|
367
499
|
}
|
|
@@ -528,14 +660,61 @@ export declare function splitTextOnTokens({ text, tokenizer }: {
|
|
|
528
660
|
tokenizer: Tokenizer;
|
|
529
661
|
}): string[];
|
|
530
662
|
|
|
663
|
+
declare const STRIP_REGEX: RegExp;
|
|
664
|
+
export { STRIP_REGEX }
|
|
665
|
+
export { STRIP_REGEX as STRIP_REGEX_alias_1 }
|
|
666
|
+
|
|
531
667
|
declare type SummaryExtractArgs = {
|
|
532
|
-
llm?:
|
|
668
|
+
llm?: MastraLanguageModel;
|
|
533
669
|
summaries?: string[];
|
|
534
670
|
promptTemplate?: SummaryPrompt['template'];
|
|
535
671
|
};
|
|
536
672
|
export { SummaryExtractArgs }
|
|
537
673
|
export { SummaryExtractArgs as SummaryExtractArgs_alias_1 }
|
|
538
674
|
|
|
675
|
+
/**
|
|
676
|
+
* Summarize an array of nodes using a custom LLM.
|
|
677
|
+
*
|
|
678
|
+
* @param nodes Array of node-like objects
|
|
679
|
+
* @param options Summary extraction options
|
|
680
|
+
* @returns Array of summary results
|
|
681
|
+
*/
|
|
682
|
+
declare class SummaryExtractor extends BaseExtractor {
|
|
683
|
+
/**
|
|
684
|
+
* MastraLanguageModel instance.
|
|
685
|
+
* @type {MastraLanguageModel}
|
|
686
|
+
*/
|
|
687
|
+
private llm;
|
|
688
|
+
/**
|
|
689
|
+
* List of summaries to extract: 'self', 'prev', 'next'
|
|
690
|
+
* @type {string[]}
|
|
691
|
+
*/
|
|
692
|
+
summaries: string[];
|
|
693
|
+
/**
|
|
694
|
+
* The prompt template to use for the summary extractor.
|
|
695
|
+
* @type {string}
|
|
696
|
+
*/
|
|
697
|
+
promptTemplate: SummaryPrompt;
|
|
698
|
+
private selfSummary;
|
|
699
|
+
private prevSummary;
|
|
700
|
+
private nextSummary;
|
|
701
|
+
constructor(options?: SummaryExtractArgs);
|
|
702
|
+
/**
|
|
703
|
+
* Extract summary from a node.
|
|
704
|
+
* @param {BaseNode} node Node to extract summary from.
|
|
705
|
+
* @returns {Promise<string>} Summary extracted from the node.
|
|
706
|
+
*/
|
|
707
|
+
generateNodeSummary(node: BaseNode): Promise<string>;
|
|
708
|
+
/**
|
|
709
|
+
* Extract summaries from a list of nodes.
|
|
710
|
+
* @param {BaseNode[]} nodes Nodes to extract summaries from.
|
|
711
|
+
* @returns {Promise<ExtractSummary[]>} Summaries extracted from the nodes.
|
|
712
|
+
*/
|
|
713
|
+
extract(nodes: BaseNode[]): Promise<ExtractSummary[]>;
|
|
714
|
+
}
|
|
715
|
+
export { SummaryExtractor }
|
|
716
|
+
export { SummaryExtractor as SummaryExtractor_alias_1 }
|
|
717
|
+
|
|
539
718
|
/**
|
|
540
719
|
* TODO: GraphRAG Enhancements
|
|
541
720
|
* - Add support for more edge types (sequential, hierarchical, citation, etc)
|
|
@@ -564,8 +743,61 @@ export declare abstract class TextTransformer implements Transformer {
|
|
|
564
743
|
protected mergeSplits(splits: string[], separator: string): string[];
|
|
565
744
|
}
|
|
566
745
|
|
|
746
|
+
/**
|
|
747
|
+
* Extract title from a list of nodes.
|
|
748
|
+
*/
|
|
749
|
+
declare class TitleExtractor extends BaseExtractor {
|
|
750
|
+
/**
|
|
751
|
+
* MastraLanguageModel instance.
|
|
752
|
+
* @type {MastraLanguageModel}
|
|
753
|
+
*/
|
|
754
|
+
llm: MastraLanguageModel;
|
|
755
|
+
/**
|
|
756
|
+
* Can work for mixture of text and non-text nodes
|
|
757
|
+
* @type {boolean}
|
|
758
|
+
* @default false
|
|
759
|
+
*/
|
|
760
|
+
isTextNodeOnly: boolean;
|
|
761
|
+
/**
|
|
762
|
+
* Number of nodes to extrct titles from.
|
|
763
|
+
* @type {number}
|
|
764
|
+
* @default 5
|
|
765
|
+
*/
|
|
766
|
+
nodes: number;
|
|
767
|
+
/**
|
|
768
|
+
* The prompt template to use for the title extractor.
|
|
769
|
+
* @type {string}
|
|
770
|
+
*/
|
|
771
|
+
nodeTemplate: TitleExtractorPrompt;
|
|
772
|
+
/**
|
|
773
|
+
* The prompt template to merge title with..
|
|
774
|
+
* @type {string}
|
|
775
|
+
*/
|
|
776
|
+
combineTemplate: TitleCombinePrompt;
|
|
777
|
+
/**
|
|
778
|
+
* Constructor for the TitleExtractor class.
|
|
779
|
+
* @param {MastraLanguageModel} llm MastraLanguageModel instance.
|
|
780
|
+
* @param {number} nodes Number of nodes to extract titles from.
|
|
781
|
+
* @param {TitleExtractorPrompt} nodeTemplate The prompt template to use for the title extractor.
|
|
782
|
+
* @param {string} combineTemplate The prompt template to merge title with..
|
|
783
|
+
*/
|
|
784
|
+
constructor(options?: TitleExtractorsArgs);
|
|
785
|
+
/**
|
|
786
|
+
* Extract titles from a list of nodes.
|
|
787
|
+
* @param {BaseNode[]} nodes Nodes to extract titles from.
|
|
788
|
+
* @returns {Promise<BaseNode<ExtractTitle>[]>} Titles extracted from the nodes.
|
|
789
|
+
*/
|
|
790
|
+
extract(nodes: BaseNode[]): Promise<Array<ExtractTitle>>;
|
|
791
|
+
private filterNodes;
|
|
792
|
+
private separateNodesByDocument;
|
|
793
|
+
private extractTitles;
|
|
794
|
+
private getTitlesCandidates;
|
|
795
|
+
}
|
|
796
|
+
export { TitleExtractor }
|
|
797
|
+
export { TitleExtractor as TitleExtractor_alias_1 }
|
|
798
|
+
|
|
567
799
|
declare type TitleExtractorsArgs = {
|
|
568
|
-
llm?:
|
|
800
|
+
llm?: MastraLanguageModel;
|
|
569
801
|
nodes?: number;
|
|
570
802
|
nodeTemplate?: TitleExtractorPrompt['template'];
|
|
571
803
|
combineTemplate?: TitleCombinePrompt['template'];
|