@vivantel/rag-core 0.1.0

package/.github/config/release-please.json

@@ -0,0 +1,38 @@
+{
+  "release-type": "node",
+  "bootstrap-sha": "main",
+  "separate-pull-requests": true,
+  "include-component-in-tag": false,
+  "tag-separator": "@",
+  "packages": {
+    ".": {
+      "package-name": "@vivantel/rag-core",
+      "changelog-path": "CHANGELOG.md",
+      "release-type": "node",
+      "bump-minor-pre-major": true,
+      "bump-patch-for-minor-pre-major": true,
+      "draft": false,
+      "prerelease": false
+    }
+  },
+  "plugins": [
+    {
+      "type": "node-workspace",
+      "merge": false
+    },
+    {
+      "type": "sentence-case",
+      "pull-request-title-pattern": "chore: release ${version}"
+    }
+  ],
+  "changelog-sections": [
+    {"type": "feat", "section": "Features", "hidden": false},
+    {"type": "fix", "section": "Bug Fixes", "hidden": false},
+    {"type": "perf", "section": "Performance Improvements", "hidden": false},
+    {"type": "refactor", "section": "Code Refactoring", "hidden": false},
+    {"type": "docs", "section": "Documentation", "hidden": false},
+    {"type": "style", "section": "Styles", "hidden": true},
+    {"type": "test", "section": "Tests", "hidden": true},
+    {"type": "chore", "section": "Miscellaneous Chores", "hidden": true}
+  ]
+}

package/.github/dependabot.yaml

@@ -0,0 +1,28 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    groups:
+      production-dependencies:
+        dependency-type: "production"
+      development-dependencies:
+        dependency-type: "development"
+    open-pull-requests-limit: 10
+    versioning-strategy: auto
+    labels:
+      - "dependencies"
+      - "automated"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+  
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "dependencies"
+      - "ci"

package/.github/workflows/ci.yaml

@@ -0,0 +1,119 @@
+name: CI
+
+on:
+  push:
+    branches: [master, develop]
+  pull_request:
+    branches: [master]
+
+jobs:
+  build-and-test:
+    name: Build & Test
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [22, 24, 26]
+      fail-fast: false
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Type check
+        run: npm run type-check
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Run tests
+        run: npm test
+
+      - name: Build
+        run: npm run build
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-node-${{ matrix.node-version }}
+          path: dist/
+          retention-days: 7
+
+  code-quality:
+    name: Code Quality
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: "lts/*"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Check formatting
+        run: npx prettier --check "src/**/*.ts"
+
+      - name: Run lint with fix check
+        run: npx eslint src/ --max-warnings 0
+
+      - name: Type coverage
+        run: npx type-coverage --at-least 90
+
+      - name: Audit dependencies
+        run: npm audit --audit-level=moderate
+
+  security:
+    name: Security Scan
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+      actions: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: "lts/*"
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@v0.36.0
+        with:
+          scan-type: "fs"
+          scan-ref: "."
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v4
+        with:
+          sarif_file: "trivy-results.sarif"
+
+      - name: Run npm audit
+        run: npm audit --json > npm-audit.json || true
+
+      - name: Upload npm audit results
+        uses: actions/upload-artifact@v4
+        with:
+          name: npm-audit
+          path: npm-audit.json

package/.github/workflows/publish.yaml

@@ -0,0 +1,151 @@
+name: Publish
+
+on:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+  packages: write
+
+jobs:
+  release-please:
+    name: Release Please
+    runs-on: ubuntu-latest
+    
+    outputs:
+      release_created: ${{ steps.release.outputs.release_created }}
+      tag_name: ${{ steps.release.outputs.tag_name }}
+      version: ${{ steps.release.outputs.version }}
+    
+    steps:
+      - name: Run Release Please
+        id: release
+        uses: googleapis/release-please-action@v5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          release-type: node
+          package-name: '@vivantel/rag-core'
+          config-file: .github/config/release-please.json
+          manifest-file: .release-please-manifest.json
+
+  publish-npm:
+    name: Publish to npm
+    needs: release-please
+    if: ${{ needs.release-please.outputs.release_created == 'true' }}
+    runs-on: ubuntu-latest
+    
+    permissions:
+      contents: read
+      id-token: write  # для npm provenance
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.release-please.outputs.tag_name }}
+      
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 'lts/*'
+          registry-url: 'https://registry.npmjs.org'
+          cache: 'npm'
+      
+      - name: Install dependencies
+        run: npm ci
+      
+      - name: Build
+        run: npm run build
+      
+      - name: Run tests
+        run: npm test
+      
+      - name: Check package size
+        run: npx package-size
+      
+      - name: Publish to npm
+        run: npm publish --provenance --access public
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ needs.release-please.outputs.tag_name }}
+          name: Release ${{ needs.release-please.outputs.version }}
+          body_path: CHANGELOG.md
+          draft: false
+          prerelease: false
+          generate_release_notes: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  create-github-package:
+    name: Publish to GitHub Packages
+    needs: release-please
+    if: ${{ needs.release-please.outputs.release_created == 'true' }}
+    runs-on: ubuntu-latest
+    
+    permissions:
+      contents: read
+      packages: write
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.release-please.outputs.tag_name }}
+      
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 'lts/*'
+          registry-url: 'https://npm.pkg.github.com'
+          cache: 'npm'
+      
+      - name: Install dependencies
+        run: npm ci
+      
+      - name: Build
+        run: npm run build
+      
+      - name: Publish to GitHub Packages
+        run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  update-docs:
+    name: Update Documentation
+    needs: release-please
+    if: ${{ needs.release-please.outputs.release_created == 'true' }}
+    runs-on: ubuntu-latest
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.release-please.outputs.tag_name }}
+      
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 'lts/*'
+          cache: 'npm'
+      
+      - name: Install dependencies
+        run: npm ci
+      
+      - name: Generate API documentation
+        run: npm run docs:generate
+      
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/api
+          destination_dir: ${{ needs.release-please.outputs.version }}
+          keep_files: false

package/.github/workflows/release.yaml

@@ -0,0 +1,150 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+  packages: write
+
+jobs:
+  release-please:
+    name: Release Please
+    runs-on: ubuntu-latest
+    
+    outputs:
+      release_created: ${{ steps.release.outputs.release_created }}
+      tag_name: ${{ steps.release.outputs.tag_name }}
+      version: ${{ steps.release.outputs.version }}
+    
+    steps:
+      - name: Run Release Please
+        id: release
+        uses: googleapis/release-please-action@v5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          release-type: node
+          package-name: '@vivantel/rag-core'
+          config-file: .github/config/release-please.json
+          manifest-file: .release-please-manifest.json
+
+  publish-npm:
+    name: Publish to npm
+    needs: release-please
+    if: ${{ needs.release-please.outputs.release_created == 'true' }}
+    runs-on: ubuntu-latest
+    
+    permissions:
+      contents: read
+      id-token: write  # для npm provenance
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.release-please.outputs.tag_name }}
+      
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: '20.x'
+          registry-url: 'https://registry.npmjs.org'
+          cache: 'npm'
+      
+      - name: Install dependencies
+        run: npm ci
+      
+      - name: Build
+        run: npm run build
+      
+      - name: Run tests
+        run: npm test
+      
+      - name: Check package size
+        run: npx package-size
+      
+      - name: Publish to npm
+        run: npm publish --provenance --access public
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v1
+        with:
+          tag_name: ${{ needs.release-please.outputs.tag_name }}
+          name: Release ${{ needs.release-please.outputs.version }}
+          body_path: CHANGELOG.md
+          draft: false
+          prerelease: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  create-github-package:
+    name: Publish to GitHub Packages
+    needs: release-please
+    if: ${{ needs.release-please.outputs.release_created == 'true' }}
+    runs-on: ubuntu-latest
+    
+    permissions:
+      contents: read
+      packages: write
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.release-please.outputs.tag_name }}
+      
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: '20.x'
+          registry-url: 'https://npm.pkg.github.com'
+          cache: 'npm'
+      
+      - name: Install dependencies
+        run: npm ci
+      
+      - name: Build
+        run: npm run build
+      
+      - name: Publish to GitHub Packages
+        run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  update-docs:
+    name: Update Documentation
+    needs: release-please
+    if: ${{ needs.release-please.outputs.release_created == 'true' }}
+    runs-on: ubuntu-latest
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.release-please.outputs.tag_name }}
+      
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: '20.x'
+          cache: 'npm'
+      
+      - name: Install dependencies
+        run: npm ci
+      
+      - name: Generate API documentation
+        run: npm run docs:generate
+      
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/api
+          destination_dir: ${{ needs.release-please.outputs.version }}
+          keep_files: false

package/.versionrc.json

@@ -0,0 +1,19 @@
+{
+  "types": [
+    {"type": "feat", "section": "Features"},
+    {"type": "fix", "section": "Bug Fixes"},
+    {"type": "perf", "section": "Performance Improvements"},
+    {"type": "refactor", "section": "Code Refactoring"},
+    {"type": "docs", "section": "Documentation"},
+    {"type": "style", "hidden": true},
+    {"type": "chore", "hidden": true},
+    {"type": "test", "hidden": true}
+  ],
+  "releaseCommitMessageFormat": "chore: release {{currentTag}}",
+  "skip": {
+    "bump": false,
+    "changelog": false,
+    "commit": false,
+    "tag": false
+  }
+}

package/CHANGELOG.md

@@ -0,0 +1,21 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Initial release
+
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+### Security

package/README.md

@@ -0,0 +1,62 @@
+# @vivantel/rag-core
+
+Core RAG pipeline tools - universal chunking, embedding, and vector store orchestration.
+
+[![CI](https://github.com/vivantel/rag-core/actions/workflows/ci.yaml/badge.svg)](https://github.com/vivantel/rag-core/actions/workflows/ci.yaml)
+[![Release](https://github.com/vivantel/rag-core/actions/workflows/release.yaml/badge.svg)](https://github.com/vivantel/rag-core/actions/workflows/release.yaml)
+[![npm version](https://img.shields.io/npm/v/@vivantel/rag-core.svg)](https://www.npmjs.com/package/@vivantel/rag-core)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## Installation
+
+```bash
+npm install @vivantel/rag-core
+```
+
+## Quick Start
+
+### 1. Create a config file (`rag.config.ts`)
+
+```typescript
+import { defineConfig, createChunker, tokenStrategy } from '@vivantel/rag-core';
+import { readFile } from 'fs/promises';
+
+export default defineConfig({
+  chunkers: [
+    createChunker({
+      name: 'my-chunker',
+      patterns: ['docs/**/*.md'],
+      async process(content, filePath, commitHash) {
+        return [{
+          content,
+          metadata: { type: 'doc', file: filePath },
+          sourceFile: filePath,
+          commitHash
+        }];
+      }
+    })
+  ],
+  
+  embedder: new MyEmbedderProvider({ apiKey: process.env.API_KEY }),
+  vectorStore: new MyVectorStore({ url: process.env.DB_URL })
+});
+```
+
+### 2. Run the pipeline
+
+```bash
+rag-update --config rag.config.ts
+```
+
+## Built-in Strategies
+
+### Chunk Strategies
+
+- `tokenStrategy({ maxTokens, overlap })` - Split by approximate token count
+- `markdownHeadersStrategy()` - Split by Markdown headers (##, ###)
+- `semanticStrategy()` - Split by sentence boundaries
+- `wholeFileStrategy()` - One chunk per file
+
+## License
+
+MIT

package/bin/rag-update.ts

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+
+import { Command } from 'commander';
+import { config } from 'dotenv';
+import { loadConfig } from '../src/config-loader.js';
+import { Orchestrator } from '../src/core/orchestrator.js';
+
+config();
+
+const program = new Command();
+
+program
+  .name('rag-update')
+  .description('Update RAG index with latest changes')
+  .version('1.0.0')
+  .option('-c, --config <path>', 'Path to config file', './rag.config.ts')
+  .option('-f, --force', 'Force full rebuild', false)
+  .option('--skip-upload', 'Skip upload to vector store', false)
+  .option('--chunks-file <path>', 'Output path for chunks.json')
+  .option('--embeddings-file <path>', 'Output path for embeddings.json')
+  .parse();
+
+async function main() {
+  const options = program.opts();
+  
+  console.log('🚀 RAG Update Tool\n');
+  
+  try {
+    const config = await loadConfig(options.config);
+    
+    const orchestrator = new Orchestrator({
+      ...config,
+      options: {
+        ...config.options,
+        force: options.force || config.options?.force,
+        skipUpload: options.skipUpload || config.options?.skipUpload,
+        chunksFile: options.chunksFile || config.options?.chunksFile,
+        embeddingsFile: options.embeddingsFile || config.options?.embeddingsFile
+      }
+    });
+    
+    await orchestrator.run();
+  } catch (error) {
+    console.error('❌ Error:', error instanceof Error ? error.message : error);
+    process.exit(1);
+  }
+}
+
+main();

package/dist/config-loader.d.ts

@@ -0,0 +1,3 @@
+import { RAGPipelineConfig } from "./core/orchestrator.js";
+export declare function loadConfig(configPath: string): Promise<RAGPipelineConfig>;
+//# sourceMappingURL=config-loader.d.ts.map

package/dist/config-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config-loader.d.ts","sourceRoot":"","sources":["../src/config-loader.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,wBAAsB,UAAU,CAC9B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,iBAAiB,CAAC,CAe5B"}

package/dist/config-loader.js

@@ -0,0 +1,13 @@
+import { pathToFileURL } from "url";
+export async function loadConfig(configPath) {
+    // Clear cache for hot reload
+    delete require.cache[require.resolve(configPath)];
+    const configUrl = pathToFileURL(configPath).href;
+    const configModule = await import(configUrl);
+    const config = configModule.default;
+    if (!config.chunkers || !config.embedder || !config.vectorStore) {
+        throw new Error("Invalid config: missing chunkers, embedder, or vectorStore");
+    }
+    return config;
+}
+//# sourceMappingURL=config-loader.js.map

package/dist/config-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config-loader.js","sourceRoot":"","sources":["../src/config-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAGpC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,UAAkB;IAElB,6BAA6B;IAC7B,OAAO,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAElD,MAAM,SAAS,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC;IACjD,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC;IAEpC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/core/chunk-processor.d.ts

@@ -0,0 +1,12 @@
+import { Chunk, FileChunker } from "../interfaces/index.js";
+export declare class ChunkProcessor {
+    private chunkers;
+    constructor(chunkers: FileChunker[]);
+    processFile(filePath: string, commitHash: string, chunker: FileChunker): Promise<Chunk[]>;
+    processFiles(files: string[], fileState: Map<string, {
+        commitHash: string;
+        chunker: FileChunker;
+    }>): Promise<Chunk[]>;
+    saveChunksLocal(chunks: Chunk[], outputFile: string): Promise<void>;
+}
+//# sourceMappingURL=chunk-processor.d.ts.map

package/dist/core/chunk-processor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-processor.d.ts","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAO5D,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAA2B;gBAE/B,QAAQ,EAAE,WAAW,EAAE;IAI7B,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,KAAK,EAAE,CAAC;IAiBb,YAAY,CAChB,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,WAAW,CAAA;KAAE,CAAC,GACnE,OAAO,CAAC,KAAK,EAAE,CAAC;IA+Bb,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAsB1E"}