@vivantel/rag-core 0.1.0 → 1.1.3

package/.github/workflows/release.yaml

@@ -1,150 +0,0 @@
-name: Release
-
-on:
-  push:
-    branches:
-      - master
-
-permissions:
-  contents: write
-  pull-requests: write
-  issues: write
-  packages: write
-
-jobs:
-  release-please:
-    name: Release Please
-    runs-on: ubuntu-latest
-    
-    outputs:
-      release_created: ${{ steps.release.outputs.release_created }}
-      tag_name: ${{ steps.release.outputs.tag_name }}
-      version: ${{ steps.release.outputs.version }}
-    
-    steps:
-      - name: Run Release Please
-        id: release
-        uses: googleapis/release-please-action@v5
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          release-type: node
-          package-name: '@vivantel/rag-core'
-          config-file: .github/config/release-please.json
-          manifest-file: .release-please-manifest.json
-
-  publish-npm:
-    name: Publish to npm
-    needs: release-please
-    if: ${{ needs.release-please.outputs.release_created == 'true' }}
-    runs-on: ubuntu-latest
-    
-    permissions:
-      contents: read
-      id-token: write  # для npm provenance
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-        with:
-          ref: ${{ needs.release-please.outputs.tag_name }}
-      
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: '20.x'
-          registry-url: 'https://registry.npmjs.org'
-          cache: 'npm'
-      
-      - name: Install dependencies
-        run: npm ci
-      
-      - name: Build
-        run: npm run build
-      
-      - name: Run tests
-        run: npm test
-      
-      - name: Check package size
-        run: npx package-size
-      
-      - name: Publish to npm
-        run: npm publish --provenance --access public
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-      
-      - name: Create GitHub Release
-        uses: softprops/action-gh-release@v1
-        with:
-          tag_name: ${{ needs.release-please.outputs.tag_name }}
-          name: Release ${{ needs.release-please.outputs.version }}
-          body_path: CHANGELOG.md
-          draft: false
-          prerelease: false
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  create-github-package:
-    name: Publish to GitHub Packages
-    needs: release-please
-    if: ${{ needs.release-please.outputs.release_created == 'true' }}
-    runs-on: ubuntu-latest
-    
-    permissions:
-      contents: read
-      packages: write
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-        with:
-          ref: ${{ needs.release-please.outputs.tag_name }}
-      
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: '20.x'
-          registry-url: 'https://npm.pkg.github.com'
-          cache: 'npm'
-      
-      - name: Install dependencies
-        run: npm ci
-      
-      - name: Build
-        run: npm run build
-      
-      - name: Publish to GitHub Packages
-        run: npm publish
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  update-docs:
-    name: Update Documentation
-    needs: release-please
-    if: ${{ needs.release-please.outputs.release_created == 'true' }}
-    runs-on: ubuntu-latest
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-        with:
-          ref: ${{ needs.release-please.outputs.tag_name }}
-      
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: '20.x'
-          cache: 'npm'
-      
-      - name: Install dependencies
-        run: npm ci
-      
-      - name: Generate API documentation
-        run: npm run docs:generate
-      
-      - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v4
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./docs/api
-          destination_dir: ${{ needs.release-please.outputs.version }}
-          keep_files: false

package/.versionrc.json

@@ -1,19 +0,0 @@
-{
-  "types": [
-    {"type": "feat", "section": "Features"},
-    {"type": "fix", "section": "Bug Fixes"},
-    {"type": "perf", "section": "Performance Improvements"},
-    {"type": "refactor", "section": "Code Refactoring"},
-    {"type": "docs", "section": "Documentation"},
-    {"type": "style", "hidden": true},
-    {"type": "chore", "hidden": true},
-    {"type": "test", "hidden": true}
-  ],
-  "releaseCommitMessageFormat": "chore: release {{currentTag}}",
-  "skip": {
-    "bump": false,
-    "changelog": false,
-    "commit": false,
-    "tag": false
-  }
-}

package/CHANGELOG.md

@@ -1,21 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [Unreleased]
-
-### Added
-- Initial release
-
-### Changed
-
-### Deprecated
-
-### Removed
-
-### Fixed
-
-### Security

package/bin/rag-update.ts

@@ -1,49 +0,0 @@
-#!/usr/bin/env node
-
-import { Command } from 'commander';
-import { config } from 'dotenv';
-import { loadConfig } from '../src/config-loader.js';
-import { Orchestrator } from '../src/core/orchestrator.js';
-
-config();
-
-const program = new Command();
-
-program
-  .name('rag-update')
-  .description('Update RAG index with latest changes')
-  .version('1.0.0')
-  .option('-c, --config <path>', 'Path to config file', './rag.config.ts')
-  .option('-f, --force', 'Force full rebuild', false)
-  .option('--skip-upload', 'Skip upload to vector store', false)
-  .option('--chunks-file <path>', 'Output path for chunks.json')
-  .option('--embeddings-file <path>', 'Output path for embeddings.json')
-  .parse();
-
-async function main() {
-  const options = program.opts();
-  
-  console.log('🚀 RAG Update Tool\n');
-  
-  try {
-    const config = await loadConfig(options.config);
-    
-    const orchestrator = new Orchestrator({
-      ...config,
-      options: {
-        ...config.options,
-        force: options.force || config.options?.force,
-        skipUpload: options.skipUpload || config.options?.skipUpload,
-        chunksFile: options.chunksFile || config.options?.chunksFile,
-        embeddingsFile: options.embeddingsFile || config.options?.embeddingsFile
-      }
-    });
-    
-    await orchestrator.run();
-  } catch (error) {
-    console.error('❌ Error:', error instanceof Error ? error.message : error);
-    process.exit(1);
-  }
-}
-
-main();

package/eslint.config.js

@@ -1,25 +0,0 @@
-import js from '@eslint/js';
-import tseslint from 'typescript-eslint';
-
-export default tseslint.config(
-  js.configs.recommended,
-  ...tseslint.configs.recommended,
-  {
-    ignores: [
-      'dist/**',
-      'node_modules/**',
-      'coverage/**',
-      '*.config.js',
-      '*.config.ts'
-    ]
-  },
-  {
-    files: ['src/**/*.ts'],
-    rules: {
-      '@typescript-eslint/no-explicit-any': 'warn',
-      '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
-      'no-console': 'off',
-      'no-undef': 'off'
-    }
-  }
-);

package/src/config-loader.ts

@@ -1,21 +0,0 @@
-import { pathToFileURL } from "url";
-import { RAGPipelineConfig } from "./core/orchestrator.js";
-
-export async function loadConfig(
-  configPath: string,
-): Promise<RAGPipelineConfig> {
-  // Clear cache for hot reload
-  delete require.cache[require.resolve(configPath)];
-
-  const configUrl = pathToFileURL(configPath).href;
-  const configModule = await import(configUrl);
-  const config = configModule.default;
-
-  if (!config.chunkers || !config.embedder || !config.vectorStore) {
-    throw new Error(
-      "Invalid config: missing chunkers, embedder, or vectorStore",
-    );
-  }
-
-  return config;
-}

package/src/core/chunk-processor.test.ts

@@ -1,36 +0,0 @@
-import { describe, it, expect, vi } from "vitest";
-import { ChunkProcessor } from "./chunk-processor.js";
-import { FileChunker } from "../interfaces/index.js";
-
-describe("ChunkProcessor", () => {
-  const mockChunker: FileChunker = {
-    name: "test",
-    patterns: ["**/*.txt"],
-    chunk: vi.fn().mockResolvedValue([
-      {
-        content: "test content",
-        metadata: { type: "test" },
-        sourceFile: "test.txt",
-        commitHash: "abc123",
-        contentHash: "hash123",
-      },
-    ]),
-  };
-
-  it("should be instantiable", () => {
-    const processor = new ChunkProcessor([mockChunker]);
-    expect(processor).toBeInstanceOf(ChunkProcessor);
-  });
-
-  it("should have processFile method", () => {
-    const processor = new ChunkProcessor([mockChunker]);
-    expect(processor.processFile).toBeDefined();
-    expect(typeof processor.processFile).toBe("function");
-  });
-
-  it("should have processFiles method", () => {
-    const processor = new ChunkProcessor([mockChunker]);
-    expect(processor.processFiles).toBeDefined();
-    expect(typeof processor.processFiles).toBe("function");
-  });
-});

package/src/core/chunk-processor.ts

@@ -1,92 +0,0 @@
-import { Chunk, FileChunker } from "../interfaces/index.js";
-import { createHash } from "crypto";
-
-function computeContentHash(content: string): string {
-  return createHash("sha256").update(content).digest("hex").slice(0, 16);
-}
-
-export class ChunkProcessor {
-  private chunkers: Map<string, FileChunker>;
-
-  constructor(chunkers: FileChunker[]) {
-    this.chunkers = new Map(chunkers.map((c) => [c.name, c]));
-  }
-
-  async processFile(
-    filePath: string,
-    commitHash: string,
-    chunker: FileChunker,
-  ): Promise<Chunk[]> {
-    try {
-      const chunks = await chunker.chunk(filePath, commitHash);
-
-      for (const chunk of chunks) {
-        chunk.contentHash = computeContentHash(chunk.content);
-        chunk.sourceFile = filePath;
-        chunk.commitHash = commitHash;
-      }
-
-      return chunks;
-    } catch (error) {
-      console.error(`  ❌ Error processing ${filePath}: ${error}`);
-      return [];
-    }
-  }
-
-  async processFiles(
-    files: string[],
-    fileState: Map<string, { commitHash: string; chunker: FileChunker }>,
-  ): Promise<Chunk[]> {
-    const allChunks: Chunk[] = [];
-
-    for (let i = 0; i < files.length; i++) {
-      const filePath = files[i];
-      const info = fileState.get(filePath);
-
-      if (!info) {
-        console.log(`  ⚠️ No chunker for: ${filePath}`);
-        continue;
-      }
-
-      console.log(`  [${i + 1}/${files.length}] ${filePath}`);
-
-      const chunks = await this.processFile(
-        filePath,
-        info.commitHash,
-        info.chunker,
-      );
-
-      if (chunks.length > 0) {
-        allChunks.push(...chunks);
-        console.log(`    ✅ Generated ${chunks.length} chunk(s)`);
-      } else {
-        console.log(`    ⚠️ No chunks generated (skipped)`);
-      }
-    }
-
-    return allChunks;
-  }
-
-  async saveChunksLocal(chunks: Chunk[], outputFile: string): Promise<void> {
-    const { dirname } = await import("path");
-    const { mkdir, writeFile, readFile } = await import("fs/promises");
-
-    await mkdir(dirname(outputFile), { recursive: true });
-
-    let existing: Chunk[] = [];
-    try {
-      const content = await readFile(outputFile, "utf-8");
-      existing = JSON.parse(content);
-    } catch {
-      // File doesn't exist
-    }
-
-    const processedFiles = new Set(chunks.map((c) => c.sourceFile));
-    const filtered = existing.filter((c) => !processedFiles.has(c.sourceFile));
-
-    const allChunks = [...filtered, ...chunks];
-
-    await writeFile(outputFile, JSON.stringify(allChunks, null, 2));
-    console.log(`\n💾 Saved ${allChunks.length} chunks to ${outputFile}`);
-  }
-}

package/src/core/embedder.ts

@@ -1,189 +0,0 @@
-import {
-  EmbeddingProvider,
-  EmbeddedChunk,
-  Chunk,
-} from "../interfaces/index.js";
-import { readFile, writeFile, mkdir } from "fs/promises";
-import { dirname } from "path";
-import { createHash } from "crypto";
-
-function chunkContentHash(chunk: Chunk): string {
-  if (chunk.contentHash) return chunk.contentHash;
-  return createHash("sha256").update(chunk.content).digest("hex").slice(0, 16);
-}
-
-export class EmbedderProcessor {
-  private provider: EmbeddingProvider;
-  private rateLimitMs: number;
-  private batchSize: number;
-
-  constructor(
-    provider: EmbeddingProvider,
-    options: { rateLimitMs?: number; batchSize?: number } = {},
-  ) {
-    this.provider = provider;
-    this.rateLimitMs = options.rateLimitMs ?? 500;
-    this.batchSize = options.batchSize ?? 10;
-  }
-
-  private async sleep(ms: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-
-  async embedChunk(chunk: Chunk): Promise<EmbeddedChunk> {
-    const embedding = await this.provider.embed(chunk.content);
-
-    return {
-      ...chunk,
-      embedding,
-      embeddedAt: Date.now() / 1000,
-    };
-  }
-
-  async embedBatch(chunks: Chunk[]): Promise<EmbeddedChunk[]> {
-    const results: EmbeddedChunk[] = [];
-
-    if (this.provider.embedBatch && chunks.length >= this.batchSize) {
-      const texts = chunks.map((c) => c.content);
-      const embeddings = await this.provider.embedBatch(texts);
-
-      for (let i = 0; i < chunks.length; i++) {
-        results.push({
-          ...chunks[i],
-          embedding: embeddings[i],
-          embeddedAt: Date.now() / 1000,
-        });
-      }
-    } else {
-      for (let i = 0; i < chunks.length; i++) {
-        const chunk = chunks[i];
-        const eventType =
-          (chunk.metadata.event_type as string) ||
-          (chunk.metadata.title as string) ||
-          chunk.sourceFile.split("/").pop() ||
-          "unknown";
-
-        console.log(`  [${i + 1}/${chunks.length}] ${eventType}`);
-
-        const embedded = await this.embedChunk(chunk);
-        results.push(embedded);
-
-        if (this.rateLimitMs > 0 && i < chunks.length - 1) {
-          await this.sleep(this.rateLimitMs);
-        }
-      }
-    }
-
-    return results;
-  }
-
-  async getChunksToEmbed(
-    chunksFile: string,
-    force: boolean = false,
-  ): Promise<{
-    chunksToEmbed: Chunk[];
-  }> {
-    let chunks: Chunk[];
-    try {
-      const content = await readFile(chunksFile, "utf-8");
-      chunks = JSON.parse(content);
-    } catch {
-      throw new Error(`Chunks file not found: ${chunksFile}`);
-    }
-
-    console.log(`📖 Loaded ${chunks.length} chunks from ${chunksFile}`);
-
-    if (force) {
-      console.log("  ⚠️ Force mode: embedding all chunks");
-      return { chunksToEmbed: chunks };
-    }
-
-    let existingEmbeddings: EmbeddedChunk[] = [];
-    const embeddingsFile = chunksFile.replace("chunks", "embeddings");
-    try {
-      const content = await readFile(embeddingsFile, "utf-8");
-      existingEmbeddings = JSON.parse(content);
-    } catch {
-      // No existing embeddings
-    }
-
-    const existingState = new Map<string, EmbeddedChunk>();
-    for (const emb of existingEmbeddings) {
-      const hash = emb.contentHash || chunkContentHash(emb);
-      existingState.set(hash, emb);
-    }
-
-    console.log(`📊 Existing embeddings: ${existingState.size} chunks`);
-
-    const chunksToEmbed: Chunk[] = [];
-    for (const chunk of chunks) {
-      const chunkHash = chunkContentHash(chunk);
-      if (!existingState.has(chunkHash)) {
-        chunksToEmbed.push(chunk);
-      }
-    }
-
-    return { chunksToEmbed };
-  }
-
-  async saveEmbeddings(
-    newEmbeddings: EmbeddedChunk[],
-    chunksFile: string,
-    force: boolean = false,
-  ): Promise<void> {
-    const embeddingsFile = chunksFile.replace("chunks", "embeddings");
-    await mkdir(dirname(embeddingsFile), { recursive: true });
-
-    const newByHash = new Map<string, EmbeddedChunk>();
-    for (const emb of newEmbeddings) {
-      const hash = emb.contentHash || chunkContentHash(emb);
-      newByHash.set(hash, emb);
-    }
-
-    let existing: EmbeddedChunk[] = [];
-    if (!force) {
-      try {
-        const content = await readFile(embeddingsFile, "utf-8");
-        existing = JSON.parse(content);
-      } catch {
-        // No existing embeddings
-      }
-    }
-
-    const final = force
-      ? []
-      : existing.filter((e) => {
-          const hash = e.contentHash || chunkContentHash(e);
-          return !newByHash.has(hash);
-        });
-
-    final.push(...newEmbeddings);
-
-    await writeFile(embeddingsFile, JSON.stringify(final, null, 2));
-    console.log(`\n💾 Saved ${final.length} embeddings to ${embeddingsFile}`);
-    console.log(
-      `   New: ${newEmbeddings.length}, Existing: ${final.length - newEmbeddings.length}`,
-    );
-  }
-
-  async run(
-    chunksFile: string,
-    force: boolean = false,
-  ): Promise<EmbeddedChunk[]> {
-    console.log("🔢 Starting incremental embedding generation...");
-
-    const { chunksToEmbed } = await this.getChunksToEmbed(chunksFile, force);
-
-    if (chunksToEmbed.length === 0) {
-      console.log("\n✨ No chunks need embedding.");
-      return [];
-    }
-
-    console.log(`\n📝 Need to embed ${chunksToEmbed.length} chunks`);
-
-    const newEmbeddings = await this.embedBatch(chunksToEmbed);
-    await this.saveEmbeddings(newEmbeddings, chunksFile, force);
-
-    return newEmbeddings;
-  }
-}

package/src/core/git-tracker.test.ts

@@ -1,64 +0,0 @@
-import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
-import { GitTracker } from "./git-tracker.js";
-import { FileChunker } from "../interfaces/index.js";
-import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "fs";
-import { join } from "path";
-import { tmpdir } from "os";
-import { execSync } from "child_process";
-
-describe("GitTracker", () => {
-  let testDir: string;
-  let originalCwd: string;
-
-  const mockChunker: FileChunker = {
-    name: "test",
-    patterns: ["**/*.txt", "**/*.yaml", "**/*.json"],
-    chunk: vi.fn().mockResolvedValue([]),
-  };
-
-  beforeEach(() => {
-    testDir = mkdtempSync(join(tmpdir(), "git-test-"));
-    originalCwd = process.cwd();
-    process.chdir(testDir);
-
-    mkdirSync(join(testDir, "src", "events"), { recursive: true });
-
-    writeFileSync(join(testDir, "test.txt"), "test content");
-    writeFileSync(
-      join(testDir, "src", "events", "booking.yaml"),
-      "event_type: BookingCreated",
-    );
-    writeFileSync(join(testDir, "config.json"), '{"key": "value"}');
-
-    execSync("git init", { stdio: "ignore" });
-    execSync('git config user.email "test@example.com"', { stdio: "ignore" });
-    execSync('git config user.name "Test"', { stdio: "ignore" });
-    execSync("git add .", { stdio: "ignore" });
-    execSync('git commit -m "initial"', { stdio: "ignore" });
-  });
-
-  afterEach(() => {
-    process.chdir(originalCwd);
-    rmSync(testDir, { recursive: true, force: true });
-  });
-
-  it("should be instantiable", () => {
-    const tracker = new GitTracker([mockChunker]);
-    expect(tracker).toBeInstanceOf(GitTracker);
-  });
-
-  it("should getAllTrackedFiles", async () => {
-    const tracker = new GitTracker([mockChunker]);
-    const files = await tracker.getAllTrackedFiles();
-
-    expect(files.length).toBeGreaterThan(0);
-    expect(files.some((f) => f.includes("test.txt"))).toBe(true);
-  });
-
-  it("should getCurrentState", async () => {
-    const tracker = new GitTracker([mockChunker]);
-    const state = await tracker.getCurrentState();
-
-    expect(state.size).toBeGreaterThan(0);
-  });
-});