@mastra/rag 0.0.2-alpha.0

package/CHANGELOG.md

@@ -0,0 +1,7 @@
+# @mastra/rag
+
+## 0.0.2-alpha.0
+
+### Patch Changes
+
+- 8413645: Initial release for rag

package/docker-compose.yaml

@@ -0,0 +1,18 @@
+services:
+  db:
+    image: pgvector/pgvector:pg16
+    container_name: 'rag-db'
+    ports:
+      - '5433:5432'
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
+      POSTGRES_DB: ${POSTGRES_DB:-mastra}
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+  qdrant:
+    image: qdrant/qdrant
+    ports:
+      - 6333:6333
+volumes:
+  pgdata:

package/jest.config.ts

@@ -0,0 +1,19 @@
+import { config } from 'dotenv';
+
+config();
+
+export default {
+  preset: 'ts-jest',
+  extensionsToTreatAsEsm: ['.ts'],
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        useESM: true,
+      },
+    ],
+  },
+};

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@mastra/rag",
+  "version": "0.0.2-alpha.0",
+  "description": "",
+  "type": "module",
+  "main": "dist/index.js",
+  "module": "dist/rag.esm.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/rag.esm.js"
+      },
+      "require": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@date-fns/utc": "^1.2.0",
+    "@llamaindex/core": "^0.4.10",
+    "@llamaindex/env": "^0.1.20",
+    "@paralleldrive/cuid2": "^2.2.2",
+    "@pinecone-database/pinecone": "^3.0.3",
+    "@qdrant/js-client-rest": "^1.12.0",
+    "date-fns": "^4.1.0",
+    "dotenv": "^16.3.1",
+    "drizzle-orm": "^0.36.1",
+    "llamaindex": "^0.8.15",
+    "pg": "^8.13.1",
+    "postgres": "^3.4.5",
+    "zod": "3.23.7",
+    "@mastra/core": "0.1.27-alpha.20"
+  },
+  "devDependencies": {
+    "@babel/preset-env": "^7.26.0",
+    "@babel/preset-typescript": "^7.26.0",
+    "@tsconfig/recommended": "^1.0.7",
+    "@types/node": "^22.9.0",
+    "@types/pg": "^8.11.10",
+    "drizzle-kit": "^0.28.0",
+    "dts-cli": "^2.0.5",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.5"
+  },
+  "scripts": {
+    "build": "dts build",
+    "build:dev": "dts watch",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js"
+  }
+}

package/src/document/index.test.ts

@@ -0,0 +1,229 @@
+import { describe, it, expect } from '@jest/globals';
+import dotenv from 'dotenv';
+
+import { MastraDocument } from '.';
+
+dotenv.config();
+
+const sampleMarkdown = `
+# Complete Guide to Modern Web Development
+## Introduction
+Welcome to our comprehensive guide on modern web development. This resource covers essential concepts, best practices, and tools that every developer should know in 2024.
+
+### Who This Guide Is For
+- Beginning developers looking to establish a solid foundation
+- Intermediate developers wanting to modernize their skillset
+- Senior developers seeking a refresher on current best practices
+
+## Core Concepts
+
+### 1. Frontend Development
+Modern frontend development has evolved significantly. Here are the key areas to focus on:
+
+#### HTML5 Semantic Elements
+Using semantic HTML improves:
+- Accessibility
+- SEO performance
+- Code readability
+- Maintenance
+
+\`\`\`html
+<header>
+    <nav>
+        <ul>
+            <li><a href="#home">Home</a></li>
+            <li><a href="#about">About</a></li>
+        </ul>
+    </nav>
+</header>
+\`\`\`
+
+#### CSS Best Practices
+1. Use CSS Custom Properties
+2. Implement responsive design
+3. Follow BEM methodology
+4. Optimize performance
+
+### 2. JavaScript Fundamentals
+
+JavaScript is the backbone of web development. Here's what you need to know:
+
+\`\`\`javascript
+// Modern JS features
+const exampleFunction = async () => {
+    try {
+        const response = await fetch('https://api.example.com/data');
+        const data = await response.json();
+        return data;
+    } catch (error) {
+        console.error('Error fetching data:', error);
+    }
+};
+\`\`\`
+
+#### Key Concepts:
+- Promises and async/await
+- ES6+ features
+- TypeScript integration
+- Module systems
+
+### 3. Backend Development
+
+Backend development requires understanding:
+
+1. **Server Architecture**
+   - RESTful APIs
+   - GraphQL
+   - Microservices
+
+2. **Database Management**
+   - SQL vs NoSQL
+   - Query optimization
+   - Data modeling
+
+3. **Security Considerations**
+   - Authentication
+   - Authorization
+   - Data encryption
+
+## Tools and Technologies
+
+### Essential Developer Tools
+
+| Category | Tools |
+|----------|-------|
+| Version Control | Git, GitHub |
+| Package Managers | npm, yarn |
+| Bundlers | webpack, Vite |
+| Testing | Jest, Cypress |
+
+### Framework Selection
+
+#### Frontend Frameworks
+1. React
+   - Component-based architecture
+   - Virtual DOM
+   - Large ecosystem
+
+2. Vue
+   - Progressive framework
+   - Easy learning curve
+   - Great documentation
+
+3. Angular
+   - Full-featured framework
+   - TypeScript integration
+   - Enterprise-ready
+
+#### Backend Frameworks
+- Node.js/Express
+- Django
+- Ruby on Rails
+- Spring Boot
+
+## Best Practices
+
+### Code Quality
+- Write clean, maintainable code
+- Follow SOLID principles
+- Implement proper error handling
+- Use consistent formatting
+
+### Performance Optimization
+1. Minimize HTTP requests
+2. Optimize images
+3. Implement caching
+4. Use lazy loading
+
+### Security Measures
+- Implement HTTPS
+- Sanitize user input
+- Use secure dependencies
+- Regular security audits
+
+## Deployment and DevOps
+
+### Continuous Integration/Continuous Deployment (CI/CD)
+1. Automated testing
+2. Build automation
+3. Deployment automation
+4. Monitoring and logging
+
+### Cloud Services
+- AWS
+- Google Cloud Platform
+- Azure
+- Heroku
+
+## Conclusion
+
+Remember that web development is an ever-evolving field. Stay current with:
+- Industry trends
+- New tools and frameworks
+- Security best practices
+- Performance optimization techniques
+
+### Additional Resources
+
+* [MDN Web Docs](https://developer.mozilla.org)
+* [Web.dev](https://web.dev)
+* [CSS-Tricks](https://css-tricks.com)
+* [JavaScript.info](https://javascript.info)
+
+---
+
+> "Any application that can be written in JavaScript, will eventually be written in JavaScript." - Jeff Atwood
+
+---
+
+**Note**: This guide is regularly updated to reflect current web development practices and standards.
+`;
+
+describe('MastraDocument', () => {
+  it('initialization', () => {
+    const doc = new MastraDocument({ text: 'test' });
+    expect(doc.documents).toHaveLength(1);
+    expect(doc.documents[0]?.text).toBe('test');
+  });
+
+  it('initialization with array', () => {
+    const doc = new MastraDocument([{ text: 'test' }, { text: 'test2' }]);
+    expect(doc.documents).toHaveLength(2);
+    expect(doc.documents[0]?.text).toBe('test');
+    expect(doc.documents[1]?.text).toBe('test2');
+  });
+
+  it('chunk - no metadata', async () => {
+    const doc = new MastraDocument({ text: sampleMarkdown });
+
+    const nodes = await doc.chunk({
+      strategy: {
+        chunkSize: 100,
+        chunkOverlap: 0,
+        separator: `\n`,
+        paragraphSeparator: `\n`,
+        secondaryChunkingRegex: `/(\n)/g`,
+      },
+    });
+
+    expect(nodes.length).toBe(10);
+  });
+
+  it('chunk - metadata title', async () => {
+    const doc = new MastraDocument({ text: sampleMarkdown });
+
+    const nodes = await doc.chunk({
+      parseMarkdown: true,
+      strategy: {
+        chunkSize: 100,
+        chunkOverlap: 0,
+        separator: `\n`,
+        paragraphSeparator: `\n`,
+        secondaryChunkingRegex: `/(\n)/g`,
+      },
+    });
+
+    expect(nodes[0]?.toJSON().text).toBe(`Complete Guide to Modern Web Development`);
+    expect(nodes.length).toBe(27);
+  }, 500000);
+});

package/src/document/index.ts

@@ -0,0 +1,129 @@
+import {
+  MarkdownNodeParser,
+  Document,
+  SummaryExtractor,
+  IngestionPipeline,
+  LLM,
+  TitleCombinePrompt,
+  TitleExtractor,
+  TitleExtractorPrompt,
+  SummaryPrompt,
+  QuestionsAnsweredExtractor,
+  QuestionExtractPrompt,
+  KeywordExtractor,
+  KeywordExtractPrompt,
+  SentenceSplitter,
+} from 'llamaindex';
+
+interface Tokenizer {
+  encode: (text: string) => Uint32Array;
+  decode: (tokens: Uint32Array) => string;
+}
+
+interface DocumentInitializer {
+  text: string;
+  metadata?: Record<string, any>;
+}
+
+type TitleExtractorsArgs = {
+  llm?: LLM;
+  nodes?: number;
+  nodeTemplate?: TitleExtractorPrompt['template'];
+  combineTemplate?: TitleCombinePrompt['template'];
+};
+
+type SummaryExtractArgs = {
+  llm?: LLM;
+  summaries?: string[];
+  promptTemplate?: SummaryPrompt['template'];
+};
+
+type QuestionAnswerExtractArgs = {
+  llm?: LLM;
+  questions?: number;
+  promptTemplate?: QuestionExtractPrompt['template'];
+  embeddingOnly?: boolean;
+};
+
+type KeywordExtractArgs = {
+  llm?: LLM;
+  keywords?: number;
+  promptTemplate?: KeywordExtractPrompt['template'];
+};
+
+type SplitterParams = {
+  tokenizer?: Tokenizer;
+};
+
+interface BaseSentenceParams {
+  chunkSize?: number;
+  chunkOverlap?: number;
+  separator?: string;
+  paragraphSeparator?: string;
+  secondaryChunkingRegex?: string;
+}
+
+type SentenceParam = BaseSentenceParams & SplitterParams;
+
+export class MastraDocument {
+  documents: Document[];
+  constructor(config: DocumentInitializer | DocumentInitializer[]) {
+    if (Array.isArray(config)) {
+      this.documents = config.map(({ text, metadata }) => new Document({ text, metadata }));
+    } else {
+      this.documents = [new Document(config)];
+    }
+  }
+
+  async chunk({
+    strategy,
+    metadataExtraction = {},
+    parseMarkdown,
+  }: {
+    parseMarkdown?: boolean;
+    strategy: SentenceParam;
+    metadataExtraction?: {
+      title?: TitleExtractorsArgs | boolean;
+      summary?: SummaryExtractArgs | boolean;
+      questionsAnswered?: QuestionAnswerExtractArgs | boolean;
+      keyword?: KeywordExtractArgs | boolean;
+    };
+  }) {
+    const { title, summary, questionsAnswered, keyword } = metadataExtraction;
+    const transformations = [];
+
+    if (parseMarkdown) {
+      transformations.push(new MarkdownNodeParser());
+    }
+
+    transformations.push(new SentenceSplitter(strategy));
+
+    if (typeof title !== 'undefined') {
+      transformations.push(new TitleExtractor(typeof title === 'boolean' ? {} : title));
+    }
+
+    if (typeof summary !== 'undefined') {
+      transformations.push(new SummaryExtractor(typeof summary === 'boolean' ? {} : summary));
+    }
+
+    if (typeof questionsAnswered !== 'undefined') {
+      transformations.push(
+        new QuestionsAnsweredExtractor(typeof questionsAnswered === 'boolean' ? {} : questionsAnswered),
+      );
+    }
+
+    if (typeof keyword !== 'undefined') {
+      transformations.push(new KeywordExtractor(typeof keyword === 'boolean' ? {} : keyword));
+    }
+
+    const pipeline = new IngestionPipeline({
+      transformations,
+    });
+
+    const nodes = await pipeline.run({
+      documents: this.documents,
+    });
+
+    return nodes;
+  }
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from './document';
+export * from './pg';
+export * from './pinecone';
+export * from './qdrant';