@vivantel/rag-core 1.1.2 → 2.0.0

package/src/core/orchestrator.ts

@@ -1,97 +0,0 @@
-import { GitTracker } from "./git-tracker.js";
-import { ChunkProcessor } from "./chunk-processor.js";
-import { EmbedderProcessor } from "./embedder.js";
-import { Uploader } from "./uploader.js";
-import {
-  FileChunker,
-  EmbeddingProvider,
-  VectorStore,
-} from "../interfaces/index.js";
-
-export interface RAGPipelineConfig {
-  chunkers: FileChunker[];
-  embedder: EmbeddingProvider;
-  vectorStore: VectorStore;
-  options?: {
-    chunksFile?: string;
-    embeddingsFile?: string;
-    force?: boolean;
-    skipUpload?: boolean;
-    rateLimitMs?: number;
-    batchSize?: number;
-  };
-}
-
-export class Orchestrator {
-  private config: RAGPipelineConfig;
-  private chunksFile: string;
-  private embeddingsFile: string;
-
-  constructor(config: RAGPipelineConfig) {
-    this.config = config;
-    this.chunksFile = config.options?.chunksFile || "./docs/rag/chunks.json";
-    this.embeddingsFile =
-      config.options?.embeddingsFile || "./docs/rag/embeddings.json";
-  }
-
-  async run(): Promise<void> {
-    console.log("🚀 Starting RAG pipeline...\n");
-
-    console.log("📂 Step 1: Scanning for changes...");
-    const gitTracker = new GitTracker(this.config.chunkers);
-    const currentState = await gitTracker.getCurrentState();
-
-    const previousState = new Map<string, string>();
-    const { toProcess, toDelete } =
-      await gitTracker.getChangedFiles(previousState);
-
-    if (
-      toProcess.length === 0 &&
-      toDelete.length === 0 &&
-      !this.config.options?.force
-    ) {
-      console.log("\n✨ No changes detected.");
-      return;
-    }
-
-    console.log(
-      `\n📊 Changes: ${toProcess.length} to process, ${toDelete.length} to delete\n`,
-    );
-
-    console.log("🔪 Step 2: Generating chunks...");
-    const chunkProcessor = new ChunkProcessor(this.config.chunkers);
-
-    const fileState = new Map();
-    for (const file of toProcess) {
-      const info = currentState.get(file);
-      if (info) fileState.set(file, info);
-    }
-
-    const chunks = await chunkProcessor.processFiles(toProcess, fileState);
-    await chunkProcessor.saveChunksLocal(chunks, this.chunksFile);
-
-    if (chunks.length === 0) {
-      console.log("\n⚠️ No chunks generated. Exiting.");
-      return;
-    }
-
-    console.log("\n🔢 Step 3: Generating embeddings...");
-    const embedder = new EmbedderProcessor(this.config.embedder, {
-      rateLimitMs: this.config.options?.rateLimitMs,
-      batchSize: this.config.options?.batchSize,
-    });
-
-    await embedder.run(this.chunksFile, this.config.options?.force || false);
-
-    if (!this.config.options?.skipUpload) {
-      console.log("\n📤 Step 4: Uploading to vector store...");
-      const uploader = new Uploader(this.config.vectorStore);
-      await uploader.sync(
-        this.embeddingsFile,
-        this.config.options?.force || false,
-      );
-    }
-
-    console.log("\n✨ RAG pipeline complete!");
-  }
-}

package/src/core/uploader.ts

@@ -1,123 +0,0 @@
-import {
-  VectorStore,
-  VectorDocument,
-  EmbeddedChunk,
-} from "../interfaces/index.js";
-import { readFile } from "fs/promises";
-
-export class Uploader {
-  private vectorStore: VectorStore;
-
-  constructor(vectorStore: VectorStore) {
-    this.vectorStore = vectorStore;
-  }
-
-  private chunkToDocument(
-    chunk: EmbeddedChunk,
-    collection?: string,
-  ): VectorDocument {
-    return {
-      content: chunk.content,
-      metadata: chunk.metadata,
-      embedding: chunk.embedding,
-      sourceFile: chunk.sourceFile,
-      commitHash: chunk.commitHash,
-      contentHash: chunk.contentHash!,
-      collection,
-    };
-  }
-
-  async getItemsToUpload(
-    embeddingsFile: string,
-    force: boolean = false,
-  ): Promise<{
-    toUpload: EmbeddedChunk[];
-    toDelete: string[];
-  }> {
-    let embeddings: EmbeddedChunk[];
-    try {
-      const content = await readFile(embeddingsFile, "utf-8");
-      embeddings = JSON.parse(content);
-    } catch {
-      throw new Error(`Embeddings file not found: ${embeddingsFile}`);
-    }
-
-    console.log(
-      `📖 Loaded ${embeddings.length} embeddings from ${embeddingsFile}`,
-    );
-
-    if (force) {
-      const allSourceFiles = [...new Set(embeddings.map((e) => e.sourceFile))];
-      return { toUpload: embeddings, toDelete: allSourceFiles };
-    }
-
-    const existingState = await this.vectorStore.getCurrentState();
-    const toUploadList: EmbeddedChunk[] = [];
-    const toDeleteSet = new Set<string>();
-
-    for (const emb of embeddings) {
-      const existingHash = existingState.get(emb.sourceFile);
-
-      if (!existingHash) {
-        toUploadList.push(emb);
-      } else if (existingHash !== emb.commitHash) {
-        toDeleteSet.add(emb.sourceFile);
-        toUploadList.push(emb);
-      }
-    }
-
-    return {
-      toUpload: toUploadList,
-      toDelete: [...toDeleteSet],
-    };
-  }
-
-  async sync(
-    embeddingsFile: string,
-    force: boolean = false,
-  ): Promise<{
-    uploaded: number;
-    deleted: number;
-  }> {
-    console.log("📤 Starting incremental upload...");
-
-    await this.vectorStore.initialize();
-
-    const { toUpload, toDelete } = await this.getItemsToUpload(
-      embeddingsFile,
-      force,
-    );
-
-    console.log(`\n📊 Need to upload: ${toUpload.length} documents`);
-    console.log(`   Need to delete: ${toDelete.length} files`);
-
-    if (toUpload.length === 0 && toDelete.length === 0) {
-      console.log("\n✨ No changes detected.");
-      return { uploaded: 0, deleted: 0 };
-    }
-
-    if (toDelete.length > 0) {
-      await this.vectorStore.deleteBySourceFile(toDelete);
-      console.log(`  🗑️ Deleted ${toDelete.length} obsolete documents`);
-    }
-
-    if (toUpload.length > 0) {
-      const documents = toUpload.map((e) => this.chunkToDocument(e));
-
-      const batchSize = 50;
-      for (let i = 0; i < documents.length; i += batchSize) {
-        const batch = documents.slice(i, i + batchSize);
-        await this.vectorStore.upsert(batch);
-        console.log(
-          `  ✅ Uploaded batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(documents.length / batchSize)}`,
-        );
-      }
-    }
-
-    console.log(`\n✨ Upload complete!`);
-    console.log(`   Uploaded: ${toUpload.length}`);
-    console.log(`   Deleted: ${toDelete.length}`);
-
-    return { uploaded: toUpload.length, deleted: toDelete.length };
-  }
-}

package/src/core/utils.ts

@@ -1,27 +0,0 @@
-import { createHash } from "crypto";
-
-export function computeContentHash(content: string): string {
-  return createHash("sha256").update(content).digest("hex").slice(0, 16);
-}
-
-export function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-export function batchArray<T>(array: T[], batchSize: number): T[][] {
-  const batches: T[][] = [];
-  for (let i = 0; i < array.length; i += batchSize) {
-    batches.push(array.slice(i, i + batchSize));
-  }
-  return batches;
-}
-
-export function extractFileName(filePath: string): string {
-  return filePath.split("/").pop() || filePath;
-}
-
-export function extractDirectory(filePath: string): string {
-  const parts = filePath.split("/");
-  parts.pop();
-  return parts.join("/");
-}

package/src/helpers/create-chunker.test.ts

@@ -1,31 +0,0 @@
-import { describe, it, expect, vi } from "vitest";
-import { createChunker } from "./create-chunker.js";
-
-describe("createChunker", () => {
-  it("should create a chunker with given options", () => {
-    const mockProcess = vi.fn().mockResolvedValue([]);
-
-    const chunker = createChunker({
-      name: "test-chunker",
-      patterns: ["**/*.txt"],
-      process: mockProcess,
-    });
-
-    expect(chunker.name).toBe("test-chunker");
-    expect(chunker.patterns).toEqual(["**/*.txt"]);
-    expect(chunker.chunk).toBeDefined();
-  });
-
-  it("should have canProcess method when provided", () => {
-    const mockCanProcess = vi.fn().mockResolvedValue(true);
-
-    const chunker = createChunker({
-      name: "test",
-      patterns: ["**/*.txt"],
-      process: vi.fn().mockResolvedValue([]),
-      canProcess: mockCanProcess,
-    });
-
-    expect(chunker.canProcess).toBeDefined();
-  });
-});

package/src/helpers/create-chunker.ts

@@ -1,40 +0,0 @@
-import { FileChunker, Chunk } from "../interfaces/index.js";
-
-export interface CreateChunkerOptions {
-  name: string;
-  patterns: string[];
-  process: (
-    content: string,
-    filePath: string,
-    commitHash: string,
-  ) => Promise<Chunk[]>;
-  canProcess?: (filePath: string, content?: string) => Promise<boolean>;
-}
-
-export function createChunker(options: CreateChunkerOptions): FileChunker {
-  return {
-    name: options.name,
-    patterns: options.patterns,
-
-    async chunk(filePath: string, commitHash: string): Promise<Chunk[]> {
-      const { readFile } = await import("fs/promises");
-      const content = await readFile(filePath, "utf-8");
-
-      if (options.canProcess) {
-        const canProcess = await options.canProcess(filePath, content);
-        if (!canProcess) {
-          return [];
-        }
-      }
-
-      return options.process(content, filePath, commitHash);
-    },
-
-    async canProcess(filePath: string, content?: string): Promise<boolean> {
-      if (options.canProcess) {
-        return options.canProcess(filePath, content);
-      }
-      return true;
-    },
-  };
-}

package/src/index.test.ts

@@ -1,33 +0,0 @@
-import { describe, it, expect } from "vitest";
-
-describe("@vivantel/rag-core", () => {
-  it("should export all public interfaces", async () => {
-    const module = await import("./index.js");
-
-    // Core
-    expect(module.GitTracker).toBeDefined();
-    expect(module.ChunkProcessor).toBeDefined();
-    expect(module.EmbedderProcessor).toBeDefined();
-    expect(module.Uploader).toBeDefined();
-    expect(module.Orchestrator).toBeDefined();
-
-    // Utils
-    expect(module.computeContentHash).toBeDefined();
-    expect(module.sleep).toBeDefined();
-    expect(module.batchArray).toBeDefined();
-    expect(module.extractFileName).toBeDefined();
-    expect(module.extractDirectory).toBeDefined();
-
-    // Strategies
-    expect(module.tokenStrategy).toBeDefined();
-    expect(module.markdownHeadersStrategy).toBeDefined();
-    expect(module.semanticStrategy).toBeDefined();
-    expect(module.wholeFileStrategy).toBeDefined();
-
-    // Helpers
-    expect(module.createChunker).toBeDefined();
-
-    // Config loader
-    expect(module.loadConfig).toBeDefined();
-  });
-});

package/src/index.ts

@@ -1,30 +0,0 @@
-// Interfaces
-export * from "./interfaces/index.js";
-
-// Core
-export { GitTracker } from "./core/git-tracker.js";
-export { ChunkProcessor } from "./core/chunk-processor.js";
-export { EmbedderProcessor } from "./core/embedder.js";
-export { Uploader } from "./core/uploader.js";
-export { Orchestrator, RAGPipelineConfig } from "./core/orchestrator.js";
-export {
-  computeContentHash,
-  sleep,
-  batchArray,
-  extractFileName,
-  extractDirectory,
-} from "./core/utils.js";
-
-// Strategies
-export {
-  tokenStrategy,
-  markdownHeadersStrategy,
-  semanticStrategy,
-  wholeFileStrategy,
-} from "./strategies/chunk/index.js";
-
-// Helpers
-export { createChunker } from "./helpers/create-chunker.js";
-
-// Config loader
-export { loadConfig } from "./config-loader.js";

package/src/interfaces/chunker.ts

@@ -1,59 +0,0 @@
-/**
- * Chunk interfaces - core building blocks for document processing
- */
-
-export interface Chunk {
-  /** The actual text content of the chunk */
-  content: string;
-
-  /** Metadata about this chunk (source file, type, etc.) */
-  metadata: Record<string, unknown>;
-
-  /** Original source file path */
-  sourceFile: string;
-
-  /** Git commit hash when this chunk was generated */
-  commitHash: string;
-
-  /** Optional unique hash of the content (for change detection) */
-  contentHash?: string;
-}
-
-export interface FileChunker {
-  /** Unique name of this chunker */
-  name: string;
-
-  /** Glob patterns this chunker handles */
-  patterns: string[];
-
-  /**
-   * Process a file and return chunks.
-   * Returns empty array if file should be skipped.
-   */
-  chunk(filePath: string, commitHash: string): Promise<Chunk[]>;
-
-  /**
-   * Optional: validate if this chunker can process the file
-   * (called before chunk() to filter early)
-   */
-  canProcess?(filePath: string, content?: string): Promise<boolean>;
-}
-
-export interface ChunkStrategy {
-  /** Strategy name */
-  name: string;
-
-  /** Split text into chunks according to strategy */
-  chunk(text: string, filePath?: string): Promise<Chunk[]>;
-
-  /** Optional: extract metadata without full chunking */
-  extractMetadata?(text: string, filePath?: string): Record<string, unknown>;
-}
-
-export interface ChunkTransformer {
-  /** Transformer name */
-  name: string;
-
-  /** Transform a chunk (return null to skip) */
-  transform(chunk: Chunk): Promise<Chunk | null>;
-}

package/src/interfaces/embedder.ts

@@ -1,36 +0,0 @@
-/**
- * Embedding provider interfaces
- */
-
-import { Chunk } from "./chunker.js";
-
-export interface EmbeddingProvider {
-  /** Provider name (e.g., 'github-models', 'openai') */
-  readonly name: string;
-
-  /** Embedding vector dimensions */
-  readonly dimensions: number;
-
-  /** Maximum tokens per request (optional) */
-  readonly maxTokens?: number;
-
-  /** Convert text to embedding vector */
-  embed(text: string): Promise<number[]>;
-
-  /** Batch convert (optional, for performance) */
-  embedBatch?(texts: string[]): Promise<number[][]>;
-
-  /** Check if provider is available (e.g., valid API key) */
-  healthCheck?(): Promise<boolean>;
-}
-
-export interface EmbeddingConfig {
-  provider: EmbeddingProvider;
-  batchSize?: number;
-  rateLimitMs?: number;
-}
-
-export interface EmbeddedChunk extends Chunk {
-  embedding: number[];
-  embeddedAt: number;
-}

package/src/interfaces/index.test.ts

@@ -1,9 +0,0 @@
-import { describe, it, expect } from "vitest";
-
-describe("Interfaces", () => {
-  it("should export types correctly", async () => {
-    const module = await import("./index.js");
-
-    expect(module).toBeDefined();
-  });
-});

package/src/interfaces/index.ts

@@ -1,3 +0,0 @@
-export * from "./chunker.js";
-export * from "./embedder.js";
-export * from "./vector-store.js";

package/src/interfaces/vector-store.ts

@@ -1,71 +0,0 @@
-/**
- * Vector store interfaces
- */
-
-export interface VectorDocument {
-  /** Unique ID (optional, auto-generated if not provided) */
-  id?: string;
-
-  /** Original text content */
-  content: string;
-
-  /** Metadata for filtering */
-  metadata: Record<string, unknown>;
-
-  /** Embedding vector */
-  embedding: number[];
-
-  /** Source file path (for tracking updates) */
-  sourceFile: string;
-
-  /** Git commit hash (for change detection) */
-  commitHash: string;
-
-  /** Content hash (for change detection) */
-  contentHash: string;
-
-  /** Collection name (for multi-collection stores) */
-  collection?: string;
-}
-
-export interface VectorSearchResult {
-  id: string;
-  content: string;
-  metadata: Record<string, unknown>;
-  similarity: number;
-}
-
-export interface VectorStore {
-  /** Store name */
-  readonly name: string;
-
-  /** Initialize store (create tables, indexes, etc.) */
-  initialize(): Promise<void>;
-
-  /** Insert or update documents */
-  upsert(documents: VectorDocument[]): Promise<void>;
-
-  /** Delete documents by source file */
-  deleteBySourceFile(sourceFiles: string[]): Promise<void>;
-
-  /** Get current state (sourceFile → commitHash) for change detection */
-  getCurrentState(collection?: string): Promise<Map<string, string>>;
-
-  /** Search by embedding vector */
-  search(
-    queryEmbedding: number[],
-    topK: number,
-    collection?: string,
-  ): Promise<VectorSearchResult[]>;
-
-  /** Optional: delete entire collection */
-  deleteCollection?(collection: string): Promise<void>;
-
-  /** Optional: get store statistics */
-  getStats?(): Promise<{ documentCount: number; collections: string[] }>;
-}
-
-export interface VectorStoreConfig {
-  provider: VectorStore;
-  collection?: string;
-}

package/src/strategies/chunk/index.ts

@@ -1,4 +0,0 @@
-export { tokenStrategy } from "./token.js";
-export { markdownHeadersStrategy } from "./markdown-headers.js";
-export { semanticStrategy } from "./semantic.js";
-export { wholeFileStrategy } from "./whole-file.js";

package/src/strategies/chunk/markdown-headers.test.ts

@@ -1,37 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { markdownHeadersStrategy } from "./markdown-headers.js";
-
-describe("markdownHeadersStrategy", () => {
-  const strategy = markdownHeadersStrategy({ minChunkSize: 10 });
-
-  it("should have correct name", () => {
-    expect(strategy.name).toBe("markdown-headers");
-  });
-
-  it("should split by headers", async () => {
-    const text = `# Header 1
-Content for header 1.
-
-## Header 2
-Content for header 2.
-
-### Header 3
-Content for header 3.`;
-
-    const chunks = await strategy.chunk(text);
-
-    expect(chunks.length).toBeGreaterThan(0);
-
-    for (const chunk of chunks) {
-      expect(chunk.metadata.header).toBeDefined();
-      expect(chunk.metadata.header_level).toBeDefined();
-    }
-  });
-
-  it("should handle text without headers", async () => {
-    const text = "Plain text without any markdown headers.";
-    const chunks = await strategy.chunk(text);
-
-    expect(Array.isArray(chunks)).toBe(true);
-  });
-});