@vivantel/rag-core 1.1.2 → 2.0.0

package/eslint.config.js

@@ -1,25 +0,0 @@
-import js from '@eslint/js';
-import tseslint from 'typescript-eslint';
-
-export default tseslint.config(
-  js.configs.recommended,
-  ...tseslint.configs.recommended,
-  {
-    ignores: [
-      'dist/**',
-      'node_modules/**',
-      'coverage/**',
-      '*.config.js',
-      '*.config.ts'
-    ]
-  },
-  {
-    files: ['src/**/*.ts'],
-    rules: {
-      '@typescript-eslint/no-explicit-any': 'warn',
-      '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
-      'no-console': 'off',
-      'no-undef': 'off'
-    }
-  }
-);

package/src/config-loader.ts

@@ -1,21 +0,0 @@
-import { pathToFileURL } from "url";
-import { RAGPipelineConfig } from "./core/orchestrator.js";
-
-export async function loadConfig(
-  configPath: string,
-): Promise<RAGPipelineConfig> {
-  // Clear cache for hot reload
-  delete require.cache[require.resolve(configPath)];
-
-  const configUrl = pathToFileURL(configPath).href;
-  const configModule = await import(configUrl);
-  const config = configModule.default;
-
-  if (!config.chunkers || !config.embedder || !config.vectorStore) {
-    throw new Error(
-      "Invalid config: missing chunkers, embedder, or vectorStore",
-    );
-  }
-
-  return config;
-}

package/src/core/chunk-processor.test.ts

@@ -1,36 +0,0 @@
-import { describe, it, expect, vi } from "vitest";
-import { ChunkProcessor } from "./chunk-processor.js";
-import { FileChunker } from "../interfaces/index.js";
-
-describe("ChunkProcessor", () => {
-  const mockChunker: FileChunker = {
-    name: "test",
-    patterns: ["**/*.txt"],
-    chunk: vi.fn().mockResolvedValue([
-      {
-        content: "test content",
-        metadata: { type: "test" },
-        sourceFile: "test.txt",
-        commitHash: "abc123",
-        contentHash: "hash123",
-      },
-    ]),
-  };
-
-  it("should be instantiable", () => {
-    const processor = new ChunkProcessor([mockChunker]);
-    expect(processor).toBeInstanceOf(ChunkProcessor);
-  });
-
-  it("should have processFile method", () => {
-    const processor = new ChunkProcessor([mockChunker]);
-    expect(processor.processFile).toBeDefined();
-    expect(typeof processor.processFile).toBe("function");
-  });
-
-  it("should have processFiles method", () => {
-    const processor = new ChunkProcessor([mockChunker]);
-    expect(processor.processFiles).toBeDefined();
-    expect(typeof processor.processFiles).toBe("function");
-  });
-});

package/src/core/chunk-processor.ts

@@ -1,92 +0,0 @@
-import { Chunk, FileChunker } from "../interfaces/index.js";
-import { createHash } from "crypto";
-
-function computeContentHash(content: string): string {
-  return createHash("sha256").update(content).digest("hex").slice(0, 16);
-}
-
-export class ChunkProcessor {
-  private chunkers: Map<string, FileChunker>;
-
-  constructor(chunkers: FileChunker[]) {
-    this.chunkers = new Map(chunkers.map((c) => [c.name, c]));
-  }
-
-  async processFile(
-    filePath: string,
-    commitHash: string,
-    chunker: FileChunker,
-  ): Promise<Chunk[]> {
-    try {
-      const chunks = await chunker.chunk(filePath, commitHash);
-
-      for (const chunk of chunks) {
-        chunk.contentHash = computeContentHash(chunk.content);
-        chunk.sourceFile = filePath;
-        chunk.commitHash = commitHash;
-      }
-
-      return chunks;
-    } catch (error) {
-      console.error(`  ❌ Error processing ${filePath}: ${error}`);
-      return [];
-    }
-  }
-
-  async processFiles(
-    files: string[],
-    fileState: Map<string, { commitHash: string; chunker: FileChunker }>,
-  ): Promise<Chunk[]> {
-    const allChunks: Chunk[] = [];
-
-    for (let i = 0; i < files.length; i++) {
-      const filePath = files[i];
-      const info = fileState.get(filePath);
-
-      if (!info) {
-        console.log(`  ⚠️ No chunker for: ${filePath}`);
-        continue;
-      }
-
-      console.log(`  [${i + 1}/${files.length}] ${filePath}`);
-
-      const chunks = await this.processFile(
-        filePath,
-        info.commitHash,
-        info.chunker,
-      );
-
-      if (chunks.length > 0) {
-        allChunks.push(...chunks);
-        console.log(`    ✅ Generated ${chunks.length} chunk(s)`);
-      } else {
-        console.log(`    ⚠️ No chunks generated (skipped)`);
-      }
-    }
-
-    return allChunks;
-  }
-
-  async saveChunksLocal(chunks: Chunk[], outputFile: string): Promise<void> {
-    const { dirname } = await import("path");
-    const { mkdir, writeFile, readFile } = await import("fs/promises");
-
-    await mkdir(dirname(outputFile), { recursive: true });
-
-    let existing: Chunk[] = [];
-    try {
-      const content = await readFile(outputFile, "utf-8");
-      existing = JSON.parse(content);
-    } catch {
-      // File doesn't exist
-    }
-
-    const processedFiles = new Set(chunks.map((c) => c.sourceFile));
-    const filtered = existing.filter((c) => !processedFiles.has(c.sourceFile));
-
-    const allChunks = [...filtered, ...chunks];
-
-    await writeFile(outputFile, JSON.stringify(allChunks, null, 2));
-    console.log(`\n💾 Saved ${allChunks.length} chunks to ${outputFile}`);
-  }
-}

package/src/core/embedder.ts

@@ -1,189 +0,0 @@
-import {
-  EmbeddingProvider,
-  EmbeddedChunk,
-  Chunk,
-} from "../interfaces/index.js";
-import { readFile, writeFile, mkdir } from "fs/promises";
-import { dirname } from "path";
-import { createHash } from "crypto";
-
-function chunkContentHash(chunk: Chunk): string {
-  if (chunk.contentHash) return chunk.contentHash;
-  return createHash("sha256").update(chunk.content).digest("hex").slice(0, 16);
-}
-
-export class EmbedderProcessor {
-  private provider: EmbeddingProvider;
-  private rateLimitMs: number;
-  private batchSize: number;
-
-  constructor(
-    provider: EmbeddingProvider,
-    options: { rateLimitMs?: number; batchSize?: number } = {},
-  ) {
-    this.provider = provider;
-    this.rateLimitMs = options.rateLimitMs ?? 500;
-    this.batchSize = options.batchSize ?? 10;
-  }
-
-  private async sleep(ms: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-
-  async embedChunk(chunk: Chunk): Promise<EmbeddedChunk> {
-    const embedding = await this.provider.embed(chunk.content);
-
-    return {
-      ...chunk,
-      embedding,
-      embeddedAt: Date.now() / 1000,
-    };
-  }
-
-  async embedBatch(chunks: Chunk[]): Promise<EmbeddedChunk[]> {
-    const results: EmbeddedChunk[] = [];
-
-    if (this.provider.embedBatch && chunks.length >= this.batchSize) {
-      const texts = chunks.map((c) => c.content);
-      const embeddings = await this.provider.embedBatch(texts);
-
-      for (let i = 0; i < chunks.length; i++) {
-        results.push({
-          ...chunks[i],
-          embedding: embeddings[i],
-          embeddedAt: Date.now() / 1000,
-        });
-      }
-    } else {
-      for (let i = 0; i < chunks.length; i++) {
-        const chunk = chunks[i];
-        const eventType =
-          (chunk.metadata.event_type as string) ||
-          (chunk.metadata.title as string) ||
-          chunk.sourceFile.split("/").pop() ||
-          "unknown";
-
-        console.log(`  [${i + 1}/${chunks.length}] ${eventType}`);
-
-        const embedded = await this.embedChunk(chunk);
-        results.push(embedded);
-
-        if (this.rateLimitMs > 0 && i < chunks.length - 1) {
-          await this.sleep(this.rateLimitMs);
-        }
-      }
-    }
-
-    return results;
-  }
-
-  async getChunksToEmbed(
-    chunksFile: string,
-    force: boolean = false,
-  ): Promise<{
-    chunksToEmbed: Chunk[];
-  }> {
-    let chunks: Chunk[];
-    try {
-      const content = await readFile(chunksFile, "utf-8");
-      chunks = JSON.parse(content);
-    } catch {
-      throw new Error(`Chunks file not found: ${chunksFile}`);
-    }
-
-    console.log(`📖 Loaded ${chunks.length} chunks from ${chunksFile}`);
-
-    if (force) {
-      console.log("  ⚠️ Force mode: embedding all chunks");
-      return { chunksToEmbed: chunks };
-    }
-
-    let existingEmbeddings: EmbeddedChunk[] = [];
-    const embeddingsFile = chunksFile.replace("chunks", "embeddings");
-    try {
-      const content = await readFile(embeddingsFile, "utf-8");
-      existingEmbeddings = JSON.parse(content);
-    } catch {
-      // No existing embeddings
-    }
-
-    const existingState = new Map<string, EmbeddedChunk>();
-    for (const emb of existingEmbeddings) {
-      const hash = emb.contentHash || chunkContentHash(emb);
-      existingState.set(hash, emb);
-    }
-
-    console.log(`📊 Existing embeddings: ${existingState.size} chunks`);
-
-    const chunksToEmbed: Chunk[] = [];
-    for (const chunk of chunks) {
-      const chunkHash = chunkContentHash(chunk);
-      if (!existingState.has(chunkHash)) {
-        chunksToEmbed.push(chunk);
-      }
-    }
-
-    return { chunksToEmbed };
-  }
-
-  async saveEmbeddings(
-    newEmbeddings: EmbeddedChunk[],
-    chunksFile: string,
-    force: boolean = false,
-  ): Promise<void> {
-    const embeddingsFile = chunksFile.replace("chunks", "embeddings");
-    await mkdir(dirname(embeddingsFile), { recursive: true });
-
-    const newByHash = new Map<string, EmbeddedChunk>();
-    for (const emb of newEmbeddings) {
-      const hash = emb.contentHash || chunkContentHash(emb);
-      newByHash.set(hash, emb);
-    }
-
-    let existing: EmbeddedChunk[] = [];
-    if (!force) {
-      try {
-        const content = await readFile(embeddingsFile, "utf-8");
-        existing = JSON.parse(content);
-      } catch {
-        // No existing embeddings
-      }
-    }
-
-    const final = force
-      ? []
-      : existing.filter((e) => {
-          const hash = e.contentHash || chunkContentHash(e);
-          return !newByHash.has(hash);
-        });
-
-    final.push(...newEmbeddings);
-
-    await writeFile(embeddingsFile, JSON.stringify(final, null, 2));
-    console.log(`\n💾 Saved ${final.length} embeddings to ${embeddingsFile}`);
-    console.log(
-      `   New: ${newEmbeddings.length}, Existing: ${final.length - newEmbeddings.length}`,
-    );
-  }
-
-  async run(
-    chunksFile: string,
-    force: boolean = false,
-  ): Promise<EmbeddedChunk[]> {
-    console.log("🔢 Starting incremental embedding generation...");
-
-    const { chunksToEmbed } = await this.getChunksToEmbed(chunksFile, force);
-
-    if (chunksToEmbed.length === 0) {
-      console.log("\n✨ No chunks need embedding.");
-      return [];
-    }
-
-    console.log(`\n📝 Need to embed ${chunksToEmbed.length} chunks`);
-
-    const newEmbeddings = await this.embedBatch(chunksToEmbed);
-    await this.saveEmbeddings(newEmbeddings, chunksFile, force);
-
-    return newEmbeddings;
-  }
-}

package/src/core/git-tracker.test.ts

@@ -1,64 +0,0 @@
-import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
-import { GitTracker } from "./git-tracker.js";
-import { FileChunker } from "../interfaces/index.js";
-import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "fs";
-import { join } from "path";
-import { tmpdir } from "os";
-import { execSync } from "child_process";
-
-describe("GitTracker", () => {
-  let testDir: string;
-  let originalCwd: string;
-
-  const mockChunker: FileChunker = {
-    name: "test",
-    patterns: ["**/*.txt", "**/*.yaml", "**/*.json"],
-    chunk: vi.fn().mockResolvedValue([]),
-  };
-
-  beforeEach(() => {
-    testDir = mkdtempSync(join(tmpdir(), "git-test-"));
-    originalCwd = process.cwd();
-    process.chdir(testDir);
-
-    mkdirSync(join(testDir, "src", "events"), { recursive: true });
-
-    writeFileSync(join(testDir, "test.txt"), "test content");
-    writeFileSync(
-      join(testDir, "src", "events", "booking.yaml"),
-      "event_type: BookingCreated",
-    );
-    writeFileSync(join(testDir, "config.json"), '{"key": "value"}');
-
-    execSync("git init", { stdio: "ignore" });
-    execSync('git config user.email "test@example.com"', { stdio: "ignore" });
-    execSync('git config user.name "Test"', { stdio: "ignore" });
-    execSync("git add .", { stdio: "ignore" });
-    execSync('git commit -m "initial"', { stdio: "ignore" });
-  });
-
-  afterEach(() => {
-    process.chdir(originalCwd);
-    rmSync(testDir, { recursive: true, force: true });
-  });
-
-  it("should be instantiable", () => {
-    const tracker = new GitTracker([mockChunker]);
-    expect(tracker).toBeInstanceOf(GitTracker);
-  });
-
-  it("should getAllTrackedFiles", async () => {
-    const tracker = new GitTracker([mockChunker]);
-    const files = await tracker.getAllTrackedFiles();
-
-    expect(files.length).toBeGreaterThan(0);
-    expect(files.some((f) => f.includes("test.txt"))).toBe(true);
-  });
-
-  it("should getCurrentState", async () => {
-    const tracker = new GitTracker([mockChunker]);
-    const state = await tracker.getCurrentState();
-
-    expect(state.size).toBeGreaterThan(0);
-  });
-});

package/src/core/git-tracker.ts

@@ -1,202 +0,0 @@
-import { simpleGit, SimpleGit } from "simple-git";
-import { glob } from "glob";
-import { FileChunker } from "../interfaces/index.js";
-import { minimatch } from "minimatch";
-import path from "path";
-
-const MAX_FILES_PER_BATCH = 100;
-const MAX_CMD_LEN = 32000;
-
-function batchFiles(files: string[]): string[][] {
-  const batches: string[][] = [];
-  let currentBatch: string[] = [];
-  let currentLen = 0;
-  const baseCmdLen = "git log -1 --format=%H --all -- ".length;
-
-  for (const file of files) {
-    const fileLen = file.length + 1;
-
-    if (
-      currentBatch.length >= MAX_FILES_PER_BATCH ||
-      currentLen + fileLen > MAX_CMD_LEN
-    ) {
-      if (currentBatch.length > 0) {
-        batches.push(currentBatch);
-        currentBatch = [];
-        currentLen = baseCmdLen;
-      }
-    }
-
-    currentBatch.push(file);
-    currentLen += fileLen;
-  }
-
-  if (currentBatch.length > 0) {
-    batches.push(currentBatch);
-  }
-
-  return batches;
-}
-
-export class GitTracker {
-  private git: SimpleGit;
-  private chunkers: FileChunker[];
-  private allPatterns: string[];
-  private currentHeadCache: string | null = null;
-  private uncommittedCache: boolean | null = null;
-
-  constructor(chunkers: FileChunker[]) {
-    this.git = simpleGit();
-    this.chunkers = chunkers;
-    this.allPatterns = chunkers.flatMap((c) => c.patterns);
-  }
-
-  private async getCurrentHead(): Promise<string> {
-    if (!this.currentHeadCache) {
-      try {
-        this.currentHeadCache = await this.git.revparse(["HEAD"]);
-      } catch {
-        this.currentHeadCache = "dev_0000000000000000000000000000000000000000";
-      }
-    }
-    return this.currentHeadCache;
-  }
-
-  private async hasUncommittedChanges(): Promise<boolean> {
-    if (this.uncommittedCache === null) {
-      try {
-        const status = await this.git.status();
-        this.uncommittedCache = status.files.length > 0;
-      } catch {
-        this.uncommittedCache = false;
-      }
-    }
-    return this.uncommittedCache;
-  }
-
-  private getChunkerForFile(filePath: string): FileChunker | null {
-    for (const chunker of this.chunkers) {
-      for (const pattern of chunker.patterns) {
-        if (this.matchesPattern(filePath, pattern)) {
-          return chunker;
-        }
-      }
-    }
-    return null;
-  }
-
-  private matchesPattern(filePath: string, pattern: string): boolean {
-    const normalizedPath = filePath.split(path.sep).join("/");
-    const normalizedPattern = pattern.split(path.sep).join("/");
-
-    return minimatch(normalizedPath, normalizedPattern);
-  }
-
-  async getAllTrackedFiles(): Promise<string[]> {
-    const files = await glob(this.allPatterns, { nodir: true });
-    return [...new Set(files)].sort();
-  }
-
-  async getCommitHashes(files: string[]): Promise<Map<string, string>> {
-    const commitMap = new Map<string, string>();
-    const batches = batchFiles(files);
-    const currentHead = await this.getCurrentHead();
-
-    for (const batch of batches) {
-      try {
-        const output = await this.git.raw([
-          "log",
-          "-1",
-          "--format=%H",
-          "--all",
-          "--",
-          ...batch,
-        ]);
-        const lines = output.trim().split("\n");
-
-        for (let i = 0; i < lines.length && i < batch.length; i++) {
-          const hash = lines[i].trim();
-          if (hash) {
-            commitMap.set(batch[i], hash);
-          }
-        }
-
-        for (const file of batch) {
-          if (!commitMap.has(file)) {
-            commitMap.set(file, currentHead);
-          }
-        }
-      } catch {
-        for (const file of batch) {
-          commitMap.set(file, currentHead);
-        }
-      }
-    }
-
-    return commitMap;
-  }
-
-  async getCurrentState(): Promise<
-    Map<string, { commitHash: string; chunker: FileChunker }>
-  > {
-    const allFiles = await this.getAllTrackedFiles();
-    const commitMap = await this.getCommitHashes(allFiles);
-    const hasDirty = await this.hasUncommittedChanges();
-    const currentHead = await this.getCurrentHead();
-
-    const state = new Map<
-      string,
-      { commitHash: string; chunker: FileChunker }
-    >();
-
-    for (const file of allFiles) {
-      let commitHash = commitMap.get(file) || currentHead;
-      if (hasDirty) {
-        commitHash = `${commitHash}-dirty`;
-      }
-
-      const chunker = this.getChunkerForFile(file);
-      if (chunker) {
-        state.set(file, { commitHash, chunker });
-      }
-    }
-
-    return state;
-  }
-
-  async getChangedFiles(previousState: Map<string, string>): Promise<{
-    toProcess: string[];
-    toDelete: string[];
-    unchanged: string[];
-  }> {
-    const current = await this.getCurrentState();
-    const toProcess: string[] = [];
-    const toDelete: string[] = [];
-    const unchanged: string[] = [];
-
-    for (const [filePath, info] of current) {
-      const prevHash = previousState.get(filePath);
-
-      if (!prevHash) {
-        console.log(`  🆕 New: ${filePath}`);
-        toProcess.push(filePath);
-      } else if (prevHash !== info.commitHash) {
-        console.log(
-          `  📝 Changed: ${filePath} (${prevHash.slice(0, 8)} → ${info.commitHash.slice(0, 8)})`,
-        );
-        toProcess.push(filePath);
-      } else {
-        unchanged.push(filePath);
-      }
-    }
-
-    for (const [filePath] of previousState) {
-      if (!current.has(filePath)) {
-        console.log(`  🗑️ Deleted: ${filePath}`);
-        toDelete.push(filePath);
-      }
-    }
-
-    return { toProcess, toDelete, unchanged };
-  }
-}

package/src/core/orchestrator.test.ts

@@ -1,53 +0,0 @@
-import { describe, it, expect, vi } from "vitest";
-import { Orchestrator, RAGPipelineConfig } from "./orchestrator.js";
-import {
-  FileChunker,
-  EmbeddingProvider,
-  VectorStore,
-} from "../interfaces/index.js";
-
-describe("Orchestrator", () => {
-  const mockChunker: FileChunker = {
-    name: "test",
-    patterns: ["**/*.txt"],
-    chunk: vi.fn().mockResolvedValue([]),
-  };
-
-  const mockEmbedder: EmbeddingProvider = {
-    name: "mock",
-    dimensions: 384,
-    embed: vi.fn().mockResolvedValue(new Array(384).fill(0)),
-  };
-
-  const mockVectorStore: VectorStore = {
-    name: "mock",
-    initialize: vi.fn().mockResolvedValue(undefined),
-    upsert: vi.fn().mockResolvedValue(undefined),
-    deleteBySourceFile: vi.fn().mockResolvedValue(undefined),
-    getCurrentState: vi.fn().mockResolvedValue(new Map()),
-    search: vi.fn().mockResolvedValue([]),
-  };
-
-  const mockConfig: RAGPipelineConfig = {
-    chunkers: [mockChunker],
-    embedder: mockEmbedder,
-    vectorStore: mockVectorStore,
-    options: {
-      chunksFile: "./test-chunks.json",
-      embeddingsFile: "./test-embeddings.json",
-      force: false,
-      skipUpload: true,
-    },
-  };
-
-  it("should be instantiable", () => {
-    const orchestrator = new Orchestrator(mockConfig);
-    expect(orchestrator).toBeInstanceOf(Orchestrator);
-  });
-
-  it("should have run method", () => {
-    const orchestrator = new Orchestrator(mockConfig);
-    expect(orchestrator.run).toBeDefined();
-    expect(typeof orchestrator.run).toBe("function");
-  });
-});