@mastra/rag 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/.turbo/turbo-lint.log +16 -0
- package/CHANGELOG.md +26 -0
- package/dist/_tsup-dts-rollup.d.ts +12 -12
- package/dist/index.js +8 -9
- package/eslint.config.js +6 -0
- package/package.json +7 -4
- package/src/document/document.ts +1 -1
- package/src/document/transformers/character.ts +2 -1
- package/src/document/transformers/text.ts +2 -2
- package/src/document/transformers/token.ts +2 -1
- package/src/document/transformers/transformer.ts +1 -1
- package/src/document/types.ts +8 -8
- package/src/graph-rag/index.test.ts +2 -1
- package/src/rerank/index.ts +4 -3
- package/src/tools/document-chunker.ts +1 -1
- package/src/tools/graph-rag.ts +1 -1
- package/src/tools/vector-query.ts +3 -2
- package/src/utils/vector-search.ts +3 -2
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
|
|
2
|
-
> @mastra/rag@0.1.
|
|
2
|
+
> @mastra/rag@0.1.1-alpha.0 build C:\Users\Ward\projects\mastra\mastra\packages\rag
|
|
3
3
|
> tsup src/index.ts --format esm --experimental-dts --clean --treeshake
|
|
4
4
|
|
|
5
5
|
[34mCLI[39m Building entry: src/index.ts
|
|
6
6
|
[34mCLI[39m Using tsconfig: tsconfig.json
|
|
7
7
|
[34mCLI[39m tsup v8.3.6
|
|
8
8
|
[34mTSC[39m Build start
|
|
9
|
-
[32mTSC[39m ⚡️ Build success in
|
|
9
|
+
[32mTSC[39m ⚡️ Build success in 15607ms
|
|
10
10
|
[34mDTS[39m Build start
|
|
11
11
|
[34mCLI[39m Target: es2022
|
|
12
|
-
Analysis will use the bundled TypeScript version 5.7.3
|
|
13
|
-
[36mWriting package typings: C:\Users\Ward\projects\mastra\mastra\packages\rag\dist\_tsup-dts-rollup.d.ts[39m
|
|
14
|
-
[32mDTS[39m ⚡️ Build success in 3914ms
|
|
15
12
|
[34mCLI[39m Cleaning output folder
|
|
16
13
|
[34mESM[39m Build start
|
|
17
|
-
|
|
18
|
-
[
|
|
14
|
+
Analysis will use the bundled TypeScript version 5.7.3
|
|
15
|
+
[36mWriting package typings: C:\Users\Ward\projects\mastra\mastra\packages\rag\dist\_tsup-dts-rollup.d.ts[39m
|
|
16
|
+
[32mDTS[39m ⚡️ Build success in 9490ms
|
|
17
|
+
[32mESM[39m [1mdist\index.js [22m[32m83.24 KB[39m
|
|
18
|
+
[32mESM[39m ⚡️ Build success in 9832ms
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
|
|
2
|
+
> @mastra/rag@0.1.0 lint C:\Users\Ward\projects\mastra\mastra\packages\rag
|
|
3
|
+
> eslint .
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
C:\Users\Ward\projects\mastra\mastra\packages\rag\src\document\transformers\token.ts
|
|
7
|
+
62:14 warning 'error' is defined but never used @typescript-eslint/no-unused-vars
|
|
8
|
+
118:14 warning 'error' is defined but never used @typescript-eslint/no-unused-vars
|
|
9
|
+
|
|
10
|
+
C:\Users\Ward\projects\mastra\mastra\packages\rag\src\graph-rag\index.test.ts
|
|
11
|
+
1:36 warning 'beforeAll' is defined but never used. Allowed unused vars must match /^ignored/u @typescript-eslint/no-unused-vars
|
|
12
|
+
1:47 warning 'afterAll' is defined but never used. Allowed unused vars must match /^ignored/u @typescript-eslint/no-unused-vars
|
|
13
|
+
1:69 warning 'afterEach' is defined but never used. Allowed unused vars must match /^ignored/u @typescript-eslint/no-unused-vars
|
|
14
|
+
|
|
15
|
+
✖ 5 problems (0 errors, 5 warnings)
|
|
16
|
+
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# @mastra/rag
|
|
2
2
|
|
|
3
|
+
## 0.1.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 91ef439: Add eslint and ran autofix
|
|
8
|
+
- Updated dependencies [d59f1a8]
|
|
9
|
+
- Updated dependencies [91ef439]
|
|
10
|
+
- Updated dependencies [4a25be4]
|
|
11
|
+
- Updated dependencies [bf2e88f]
|
|
12
|
+
- Updated dependencies [2f0d707]
|
|
13
|
+
- Updated dependencies [aac1667]
|
|
14
|
+
- @mastra/core@0.2.1
|
|
15
|
+
|
|
16
|
+
## 0.1.1-alpha.0
|
|
17
|
+
|
|
18
|
+
### Patch Changes
|
|
19
|
+
|
|
20
|
+
- 91ef439: Add eslint and ran autofix
|
|
21
|
+
- Updated dependencies [d59f1a8]
|
|
22
|
+
- Updated dependencies [91ef439]
|
|
23
|
+
- Updated dependencies [4a25be4]
|
|
24
|
+
- Updated dependencies [bf2e88f]
|
|
25
|
+
- Updated dependencies [2f0d707]
|
|
26
|
+
- Updated dependencies [aac1667]
|
|
27
|
+
- @mastra/core@0.2.1-alpha.0
|
|
28
|
+
|
|
3
29
|
## 0.1.0
|
|
4
30
|
|
|
5
31
|
### Minor Changes
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import { createTool } from '@mastra/core/tools';
|
|
2
2
|
import { Document as Document_2 } from 'llamaindex';
|
|
3
|
-
import { EmbeddingModel } from 'ai';
|
|
4
|
-
import { KeywordExtractPrompt } from 'llamaindex';
|
|
5
|
-
import { LanguageModelV1 } from 'ai';
|
|
6
|
-
import { LLM } from 'llamaindex';
|
|
7
|
-
import { MastraVector } from '@mastra/core/vector';
|
|
8
|
-
import { QueryResult } from '@mastra/core/vector';
|
|
9
|
-
import { QuestionExtractPrompt } from 'llamaindex';
|
|
10
|
-
import { SummaryPrompt } from 'llamaindex';
|
|
11
|
-
import { TiktokenEncoding } from 'js-tiktoken';
|
|
12
|
-
import { TiktokenModel } from 'js-tiktoken';
|
|
13
|
-
import { TitleCombinePrompt } from 'llamaindex';
|
|
14
|
-
import { TitleExtractorPrompt } from 'llamaindex';
|
|
3
|
+
import type { EmbeddingModel } from 'ai';
|
|
4
|
+
import type { KeywordExtractPrompt } from 'llamaindex';
|
|
5
|
+
import type { LanguageModelV1 } from 'ai';
|
|
6
|
+
import type { LLM } from 'llamaindex';
|
|
7
|
+
import type { MastraVector } from '@mastra/core/vector';
|
|
8
|
+
import type { QueryResult } from '@mastra/core/vector';
|
|
9
|
+
import type { QuestionExtractPrompt } from 'llamaindex';
|
|
10
|
+
import type { SummaryPrompt } from 'llamaindex';
|
|
11
|
+
import type { TiktokenEncoding } from 'js-tiktoken';
|
|
12
|
+
import type { TiktokenModel } from 'js-tiktoken';
|
|
13
|
+
import type { TitleCombinePrompt } from 'llamaindex';
|
|
14
|
+
import type { TitleExtractorPrompt } from 'llamaindex';
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* Vector store specific prompts that detail supported operators and examples.
|
package/dist/index.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { Document, SummaryExtractor, QuestionsAnsweredExtractor, KeywordExtractor, TitleExtractor, IngestionPipeline } from 'llamaindex';
|
|
2
|
-
import { encodingForModel, getEncoding } from 'js-tiktoken';
|
|
3
2
|
import { parse } from 'node-html-better-parser';
|
|
3
|
+
import { encodingForModel, getEncoding } from 'js-tiktoken';
|
|
4
4
|
import { CohereRelevanceScorer, MastraAgentRelevanceScorer } from '@mastra/core/relevance';
|
|
5
|
-
import '@mastra/core/vector';
|
|
6
|
-
import { embed } from 'ai';
|
|
7
5
|
import { createTool } from '@mastra/core/tools';
|
|
8
6
|
import { z } from 'zod';
|
|
7
|
+
import { embed } from 'ai';
|
|
9
8
|
|
|
10
9
|
// src/document/document.ts
|
|
10
|
+
|
|
11
|
+
// src/document/types.ts
|
|
11
12
|
var Language = /* @__PURE__ */ ((Language2) => {
|
|
12
13
|
Language2["CPP"] = "cpp";
|
|
13
14
|
Language2["GO"] = "go";
|
|
@@ -37,8 +38,6 @@ var Language = /* @__PURE__ */ ((Language2) => {
|
|
|
37
38
|
Language2["POWERSHELL"] = "powershell";
|
|
38
39
|
return Language2;
|
|
39
40
|
})(Language || {});
|
|
40
|
-
|
|
41
|
-
// src/document/transformers/text.ts
|
|
42
41
|
var TextTransformer = class {
|
|
43
42
|
size;
|
|
44
43
|
overlap;
|
|
@@ -849,7 +848,7 @@ var MarkdownHeaderTransformer = class {
|
|
|
849
848
|
const aggChunk = aggregatedChunks[aggregatedChunks.length - 1];
|
|
850
849
|
aggChunk.content += " \n" + line.content;
|
|
851
850
|
} else if (aggregatedChunks.length > 0 && JSON.stringify(aggregatedChunks?.[aggregatedChunks.length - 1].metadata) !== JSON.stringify(line.metadata) && Object.keys(aggregatedChunks?.[aggregatedChunks.length - 1].metadata).length < Object.keys(line.metadata).length && aggregatedChunks?.[aggregatedChunks.length - 1]?.content?.split("\n")?.slice(-1)[0][0] === "#" && !this.stripHeaders) {
|
|
852
|
-
if (aggregatedChunks?.[aggregatedChunks.length - 1]) {
|
|
851
|
+
if (aggregatedChunks && aggregatedChunks?.[aggregatedChunks.length - 1]) {
|
|
853
852
|
const aggChunk = aggregatedChunks[aggregatedChunks.length - 1];
|
|
854
853
|
if (aggChunk) {
|
|
855
854
|
aggChunk.content += " \n" + line.content;
|
|
@@ -1424,7 +1423,7 @@ var GraphRAG = class {
|
|
|
1424
1423
|
return this.edges.filter((edge) => edge.source === nodeId && (!edgeType || edge.type === edgeType)).map((edge) => ({
|
|
1425
1424
|
id: edge.target,
|
|
1426
1425
|
weight: edge.weight
|
|
1427
|
-
})).filter((node) => node !==
|
|
1426
|
+
})).filter((node) => node !== void 0);
|
|
1428
1427
|
}
|
|
1429
1428
|
// Calculate cosine similarity between two vectors
|
|
1430
1429
|
cosineSimilarity(vec1, vec2) {
|
|
@@ -1686,7 +1685,7 @@ var createGraphRAGTool = ({
|
|
|
1686
1685
|
vectorStore,
|
|
1687
1686
|
queryText,
|
|
1688
1687
|
model,
|
|
1689
|
-
queryFilter: Object.keys(queryFilter || {}).length > 0 ? queryFilter :
|
|
1688
|
+
queryFilter: Object.keys(queryFilter || {}).length > 0 ? queryFilter : void 0,
|
|
1690
1689
|
topK,
|
|
1691
1690
|
includeVectors: true
|
|
1692
1691
|
});
|
|
@@ -1761,7 +1760,7 @@ var createVectorQueryTool = ({
|
|
|
1761
1760
|
vectorStore,
|
|
1762
1761
|
queryText,
|
|
1763
1762
|
model,
|
|
1764
|
-
queryFilter: Object.keys(queryFilter || {}).length > 0 ? queryFilter :
|
|
1763
|
+
queryFilter: Object.keys(queryFilter || {}).length > 0 ? queryFilter : void 0,
|
|
1765
1764
|
topK
|
|
1766
1765
|
});
|
|
1767
1766
|
if (reranker) {
|
package/eslint.config.js
ADDED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mastra/rag",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"node-html-better-parser": "^1.4.7",
|
|
26
26
|
"pathe": "^2.0.2",
|
|
27
27
|
"zod": "^3.24.1",
|
|
28
|
-
"@mastra/core": "^0.2.
|
|
28
|
+
"@mastra/core": "^0.2.1"
|
|
29
29
|
},
|
|
30
30
|
"peerDependencies": {
|
|
31
31
|
"ai": "^4.0.0"
|
|
@@ -39,7 +39,9 @@
|
|
|
39
39
|
"dotenv": "^16.4.7",
|
|
40
40
|
"tsup": "^8.0.1",
|
|
41
41
|
"typescript": "^5.7.3",
|
|
42
|
-
"vitest": "^3.0.4"
|
|
42
|
+
"vitest": "^3.0.4",
|
|
43
|
+
"eslint": "^9.20.1",
|
|
44
|
+
"@internal/lint": "0.0.0"
|
|
43
45
|
},
|
|
44
46
|
"keywords": [
|
|
45
47
|
"rag",
|
|
@@ -60,6 +62,7 @@
|
|
|
60
62
|
"scripts": {
|
|
61
63
|
"build": "tsup src/index.ts --format esm --experimental-dts --clean --treeshake",
|
|
62
64
|
"buld:watch": "pnpm build --watch",
|
|
63
|
-
"test": "vitest run"
|
|
65
|
+
"test": "vitest run",
|
|
66
|
+
"lint": "eslint ."
|
|
64
67
|
}
|
|
65
68
|
}
|
package/src/document/document.ts
CHANGED
|
@@ -13,7 +13,7 @@ import { RecursiveJsonTransformer } from './transformers/json';
|
|
|
13
13
|
import { LatexTransformer } from './transformers/latex';
|
|
14
14
|
import { MarkdownHeaderTransformer, MarkdownTransformer } from './transformers/markdown';
|
|
15
15
|
import { TokenTransformer } from './transformers/token';
|
|
16
|
-
import {
|
|
16
|
+
import type { ChunkOptions, ChunkParams, ChunkStrategy, ExtractParams } from './types';
|
|
17
17
|
|
|
18
18
|
export class MDocument {
|
|
19
19
|
private chunks: Chunk[];
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Document } from 'llamaindex';
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import type { ChunkOptions } from '../types';
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import type { Transformer } from './transformer';
|
|
6
6
|
|
|
7
7
|
export abstract class TextTransformer implements Transformer {
|
|
8
8
|
protected size: number;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { TiktokenModel, TiktokenEncoding, Tiktoken } from 'js-tiktoken';
|
|
2
|
+
import { encodingForModel, getEncoding } from 'js-tiktoken';
|
|
2
3
|
|
|
3
4
|
import { TextTransformer } from './text';
|
|
4
5
|
|
package/src/document/types.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
1
|
+
import type { TiktokenEncoding, TiktokenModel } from 'js-tiktoken';
|
|
2
|
+
import type {
|
|
3
|
+
LLM,
|
|
4
|
+
TitleCombinePrompt,
|
|
5
|
+
TitleExtractorPrompt,
|
|
6
|
+
SummaryPrompt,
|
|
7
|
+
QuestionExtractPrompt,
|
|
8
|
+
KeywordExtractPrompt,
|
|
9
9
|
} from 'llamaindex';
|
|
10
10
|
|
|
11
11
|
export enum Language {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { describe, it, expect, vi, beforeAll, afterAll, beforeEach, afterEach } from 'vitest';
|
|
2
2
|
|
|
3
|
-
import { GraphChunk, GraphEdge, GraphEmbedding, GraphNode
|
|
3
|
+
import type { GraphChunk, GraphEdge, GraphEmbedding, GraphNode } from './';
|
|
4
|
+
import { GraphRAG } from './';
|
|
4
5
|
|
|
5
6
|
describe('GraphRAG', () => {
|
|
6
7
|
beforeEach(() => {
|
package/src/rerank/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { MastraAgentRelevanceScorer, CohereRelevanceScorer
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { MastraAgentRelevanceScorer, CohereRelevanceScorer } from '@mastra/core/relevance';
|
|
2
|
+
import type { RelevanceScoreProvider } from '@mastra/core/relevance';
|
|
3
|
+
import type { QueryResult } from '@mastra/core/vector';
|
|
4
|
+
import type { LanguageModelV1 } from 'ai';
|
|
4
5
|
|
|
5
6
|
// Default weights for different scoring components (must add up to 1)
|
|
6
7
|
const DEFAULT_WEIGHTS = {
|
package/src/tools/graph-rag.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { createTool } from '@mastra/core/tools';
|
|
2
|
-
import {
|
|
2
|
+
import type { EmbeddingModel } from 'ai';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
|
|
5
|
-
import { rerank
|
|
5
|
+
import { rerank } from '../rerank';
|
|
6
|
+
import type { RerankConfig } from '../rerank';
|
|
6
7
|
import { vectorQuerySearch, defaultVectorQueryDescription } from '../utils';
|
|
7
8
|
|
|
8
9
|
export const createVectorQueryTool = ({
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { embed
|
|
1
|
+
import type { MastraVector, QueryResult } from '@mastra/core/vector';
|
|
2
|
+
import { embed } from 'ai';
|
|
3
|
+
import type { EmbeddingModel } from 'ai';
|
|
3
4
|
|
|
4
5
|
interface VectorQuerySearchParams {
|
|
5
6
|
indexName: string;
|