@vivantel/rag-core 0.1.0 → 1.1.3

package/src/core/git-tracker.ts

@@ -1,202 +0,0 @@
-import { simpleGit, SimpleGit } from "simple-git";
-import { glob } from "glob";
-import { FileChunker } from "../interfaces/index.js";
-import { minimatch } from "minimatch";
-import path from "path";
-
-const MAX_FILES_PER_BATCH = 100;
-const MAX_CMD_LEN = 32000;
-
-function batchFiles(files: string[]): string[][] {
-  const batches: string[][] = [];
-  let currentBatch: string[] = [];
-  let currentLen = 0;
-  const baseCmdLen = "git log -1 --format=%H --all -- ".length;
-
-  for (const file of files) {
-    const fileLen = file.length + 1;
-
-    if (
-      currentBatch.length >= MAX_FILES_PER_BATCH ||
-      currentLen + fileLen > MAX_CMD_LEN
-    ) {
-      if (currentBatch.length > 0) {
-        batches.push(currentBatch);
-        currentBatch = [];
-        currentLen = baseCmdLen;
-      }
-    }
-
-    currentBatch.push(file);
-    currentLen += fileLen;
-  }
-
-  if (currentBatch.length > 0) {
-    batches.push(currentBatch);
-  }
-
-  return batches;
-}
-
-export class GitTracker {
-  private git: SimpleGit;
-  private chunkers: FileChunker[];
-  private allPatterns: string[];
-  private currentHeadCache: string | null = null;
-  private uncommittedCache: boolean | null = null;
-
-  constructor(chunkers: FileChunker[]) {
-    this.git = simpleGit();
-    this.chunkers = chunkers;
-    this.allPatterns = chunkers.flatMap((c) => c.patterns);
-  }
-
-  private async getCurrentHead(): Promise<string> {
-    if (!this.currentHeadCache) {
-      try {
-        this.currentHeadCache = await this.git.revparse(["HEAD"]);
-      } catch {
-        this.currentHeadCache = "dev_0000000000000000000000000000000000000000";
-      }
-    }
-    return this.currentHeadCache;
-  }
-
-  private async hasUncommittedChanges(): Promise<boolean> {
-    if (this.uncommittedCache === null) {
-      try {
-        const status = await this.git.status();
-        this.uncommittedCache = status.files.length > 0;
-      } catch {
-        this.uncommittedCache = false;
-      }
-    }
-    return this.uncommittedCache;
-  }
-
-  private getChunkerForFile(filePath: string): FileChunker | null {
-    for (const chunker of this.chunkers) {
-      for (const pattern of chunker.patterns) {
-        if (this.matchesPattern(filePath, pattern)) {
-          return chunker;
-        }
-      }
-    }
-    return null;
-  }
-
-  private matchesPattern(filePath: string, pattern: string): boolean {
-    const normalizedPath = filePath.split(path.sep).join("/");
-    const normalizedPattern = pattern.split(path.sep).join("/");
-
-    return minimatch(normalizedPath, normalizedPattern);
-  }
-
-  async getAllTrackedFiles(): Promise<string[]> {
-    const files = await glob(this.allPatterns, { nodir: true });
-    return [...new Set(files)].sort();
-  }
-
-  async getCommitHashes(files: string[]): Promise<Map<string, string>> {
-    const commitMap = new Map<string, string>();
-    const batches = batchFiles(files);
-    const currentHead = await this.getCurrentHead();
-
-    for (const batch of batches) {
-      try {
-        const output = await this.git.raw([
-          "log",
-          "-1",
-          "--format=%H",
-          "--all",
-          "--",
-          ...batch,
-        ]);
-        const lines = output.trim().split("\n");
-
-        for (let i = 0; i < lines.length && i < batch.length; i++) {
-          const hash = lines[i].trim();
-          if (hash) {
-            commitMap.set(batch[i], hash);
-          }
-        }
-
-        for (const file of batch) {
-          if (!commitMap.has(file)) {
-            commitMap.set(file, currentHead);
-          }
-        }
-      } catch {
-        for (const file of batch) {
-          commitMap.set(file, currentHead);
-        }
-      }
-    }
-
-    return commitMap;
-  }
-
-  async getCurrentState(): Promise<
-    Map<string, { commitHash: string; chunker: FileChunker }>
-  > {
-    const allFiles = await this.getAllTrackedFiles();
-    const commitMap = await this.getCommitHashes(allFiles);
-    const hasDirty = await this.hasUncommittedChanges();
-    const currentHead = await this.getCurrentHead();
-
-    const state = new Map<
-      string,
-      { commitHash: string; chunker: FileChunker }
-    >();
-
-    for (const file of allFiles) {
-      let commitHash = commitMap.get(file) || currentHead;
-      if (hasDirty) {
-        commitHash = `${commitHash}-dirty`;
-      }
-
-      const chunker = this.getChunkerForFile(file);
-      if (chunker) {
-        state.set(file, { commitHash, chunker });
-      }
-    }
-
-    return state;
-  }
-
-  async getChangedFiles(previousState: Map<string, string>): Promise<{
-    toProcess: string[];
-    toDelete: string[];
-    unchanged: string[];
-  }> {
-    const current = await this.getCurrentState();
-    const toProcess: string[] = [];
-    const toDelete: string[] = [];
-    const unchanged: string[] = [];
-
-    for (const [filePath, info] of current) {
-      const prevHash = previousState.get(filePath);
-
-      if (!prevHash) {
-        console.log(`  🆕 New: ${filePath}`);
-        toProcess.push(filePath);
-      } else if (prevHash !== info.commitHash) {
-        console.log(
-          `  📝 Changed: ${filePath} (${prevHash.slice(0, 8)} → ${info.commitHash.slice(0, 8)})`,
-        );
-        toProcess.push(filePath);
-      } else {
-        unchanged.push(filePath);
-      }
-    }
-
-    for (const [filePath] of previousState) {
-      if (!current.has(filePath)) {
-        console.log(`  🗑️ Deleted: ${filePath}`);
-        toDelete.push(filePath);
-      }
-    }
-
-    return { toProcess, toDelete, unchanged };
-  }
-}

package/src/core/orchestrator.test.ts

@@ -1,53 +0,0 @@
-import { describe, it, expect, vi } from "vitest";
-import { Orchestrator, RAGPipelineConfig } from "./orchestrator.js";
-import {
-  FileChunker,
-  EmbeddingProvider,
-  VectorStore,
-} from "../interfaces/index.js";
-
-describe("Orchestrator", () => {
-  const mockChunker: FileChunker = {
-    name: "test",
-    patterns: ["**/*.txt"],
-    chunk: vi.fn().mockResolvedValue([]),
-  };
-
-  const mockEmbedder: EmbeddingProvider = {
-    name: "mock",
-    dimensions: 384,
-    embed: vi.fn().mockResolvedValue(new Array(384).fill(0)),
-  };
-
-  const mockVectorStore: VectorStore = {
-    name: "mock",
-    initialize: vi.fn().mockResolvedValue(undefined),
-    upsert: vi.fn().mockResolvedValue(undefined),
-    deleteBySourceFile: vi.fn().mockResolvedValue(undefined),
-    getCurrentState: vi.fn().mockResolvedValue(new Map()),
-    search: vi.fn().mockResolvedValue([]),
-  };
-
-  const mockConfig: RAGPipelineConfig = {
-    chunkers: [mockChunker],
-    embedder: mockEmbedder,
-    vectorStore: mockVectorStore,
-    options: {
-      chunksFile: "./test-chunks.json",
-      embeddingsFile: "./test-embeddings.json",
-      force: false,
-      skipUpload: true,
-    },
-  };
-
-  it("should be instantiable", () => {
-    const orchestrator = new Orchestrator(mockConfig);
-    expect(orchestrator).toBeInstanceOf(Orchestrator);
-  });
-
-  it("should have run method", () => {
-    const orchestrator = new Orchestrator(mockConfig);
-    expect(orchestrator.run).toBeDefined();
-    expect(typeof orchestrator.run).toBe("function");
-  });
-});

package/src/core/orchestrator.ts

@@ -1,97 +0,0 @@
-import { GitTracker } from "./git-tracker.js";
-import { ChunkProcessor } from "./chunk-processor.js";
-import { EmbedderProcessor } from "./embedder.js";
-import { Uploader } from "./uploader.js";
-import {
-  FileChunker,
-  EmbeddingProvider,
-  VectorStore,
-} from "../interfaces/index.js";
-
-export interface RAGPipelineConfig {
-  chunkers: FileChunker[];
-  embedder: EmbeddingProvider;
-  vectorStore: VectorStore;
-  options?: {
-    chunksFile?: string;
-    embeddingsFile?: string;
-    force?: boolean;
-    skipUpload?: boolean;
-    rateLimitMs?: number;
-    batchSize?: number;
-  };
-}
-
-export class Orchestrator {
-  private config: RAGPipelineConfig;
-  private chunksFile: string;
-  private embeddingsFile: string;
-
-  constructor(config: RAGPipelineConfig) {
-    this.config = config;
-    this.chunksFile = config.options?.chunksFile || "./docs/rag/chunks.json";
-    this.embeddingsFile =
-      config.options?.embeddingsFile || "./docs/rag/embeddings.json";
-  }
-
-  async run(): Promise<void> {
-    console.log("🚀 Starting RAG pipeline...\n");
-
-    console.log("📂 Step 1: Scanning for changes...");
-    const gitTracker = new GitTracker(this.config.chunkers);
-    const currentState = await gitTracker.getCurrentState();
-
-    const previousState = new Map<string, string>();
-    const { toProcess, toDelete } =
-      await gitTracker.getChangedFiles(previousState);
-
-    if (
-      toProcess.length === 0 &&
-      toDelete.length === 0 &&
-      !this.config.options?.force
-    ) {
-      console.log("\n✨ No changes detected.");
-      return;
-    }
-
-    console.log(
-      `\n📊 Changes: ${toProcess.length} to process, ${toDelete.length} to delete\n`,
-    );
-
-    console.log("🔪 Step 2: Generating chunks...");
-    const chunkProcessor = new ChunkProcessor(this.config.chunkers);
-
-    const fileState = new Map();
-    for (const file of toProcess) {
-      const info = currentState.get(file);
-      if (info) fileState.set(file, info);
-    }
-
-    const chunks = await chunkProcessor.processFiles(toProcess, fileState);
-    await chunkProcessor.saveChunksLocal(chunks, this.chunksFile);
-
-    if (chunks.length === 0) {
-      console.log("\n⚠️ No chunks generated. Exiting.");
-      return;
-    }
-
-    console.log("\n🔢 Step 3: Generating embeddings...");
-    const embedder = new EmbedderProcessor(this.config.embedder, {
-      rateLimitMs: this.config.options?.rateLimitMs,
-      batchSize: this.config.options?.batchSize,
-    });
-
-    await embedder.run(this.chunksFile, this.config.options?.force || false);
-
-    if (!this.config.options?.skipUpload) {
-      console.log("\n📤 Step 4: Uploading to vector store...");
-      const uploader = new Uploader(this.config.vectorStore);
-      await uploader.sync(
-        this.embeddingsFile,
-        this.config.options?.force || false,
-      );
-    }
-
-    console.log("\n✨ RAG pipeline complete!");
-  }
-}

package/src/core/uploader.ts

@@ -1,123 +0,0 @@
-import {
-  VectorStore,
-  VectorDocument,
-  EmbeddedChunk,
-} from "../interfaces/index.js";
-import { readFile } from "fs/promises";
-
-export class Uploader {
-  private vectorStore: VectorStore;
-
-  constructor(vectorStore: VectorStore) {
-    this.vectorStore = vectorStore;
-  }
-
-  private chunkToDocument(
-    chunk: EmbeddedChunk,
-    collection?: string,
-  ): VectorDocument {
-    return {
-      content: chunk.content,
-      metadata: chunk.metadata,
-      embedding: chunk.embedding,
-      sourceFile: chunk.sourceFile,
-      commitHash: chunk.commitHash,
-      contentHash: chunk.contentHash!,
-      collection,
-    };
-  }
-
-  async getItemsToUpload(
-    embeddingsFile: string,
-    force: boolean = false,
-  ): Promise<{
-    toUpload: EmbeddedChunk[];
-    toDelete: string[];
-  }> {
-    let embeddings: EmbeddedChunk[];
-    try {
-      const content = await readFile(embeddingsFile, "utf-8");
-      embeddings = JSON.parse(content);
-    } catch {
-      throw new Error(`Embeddings file not found: ${embeddingsFile}`);
-    }
-
-    console.log(
-      `📖 Loaded ${embeddings.length} embeddings from ${embeddingsFile}`,
-    );
-
-    if (force) {
-      const allSourceFiles = [...new Set(embeddings.map((e) => e.sourceFile))];
-      return { toUpload: embeddings, toDelete: allSourceFiles };
-    }
-
-    const existingState = await this.vectorStore.getCurrentState();
-    const toUploadList: EmbeddedChunk[] = [];
-    const toDeleteSet = new Set<string>();
-
-    for (const emb of embeddings) {
-      const existingHash = existingState.get(emb.sourceFile);
-
-      if (!existingHash) {
-        toUploadList.push(emb);
-      } else if (existingHash !== emb.commitHash) {
-        toDeleteSet.add(emb.sourceFile);
-        toUploadList.push(emb);
-      }
-    }
-
-    return {
-      toUpload: toUploadList,
-      toDelete: [...toDeleteSet],
-    };
-  }
-
-  async sync(
-    embeddingsFile: string,
-    force: boolean = false,
-  ): Promise<{
-    uploaded: number;
-    deleted: number;
-  }> {
-    console.log("📤 Starting incremental upload...");
-
-    await this.vectorStore.initialize();
-
-    const { toUpload, toDelete } = await this.getItemsToUpload(
-      embeddingsFile,
-      force,
-    );
-
-    console.log(`\n📊 Need to upload: ${toUpload.length} documents`);
-    console.log(`   Need to delete: ${toDelete.length} files`);
-
-    if (toUpload.length === 0 && toDelete.length === 0) {
-      console.log("\n✨ No changes detected.");
-      return { uploaded: 0, deleted: 0 };
-    }
-
-    if (toDelete.length > 0) {
-      await this.vectorStore.deleteBySourceFile(toDelete);
-      console.log(`  🗑️ Deleted ${toDelete.length} obsolete documents`);
-    }
-
-    if (toUpload.length > 0) {
-      const documents = toUpload.map((e) => this.chunkToDocument(e));
-
-      const batchSize = 50;
-      for (let i = 0; i < documents.length; i += batchSize) {
-        const batch = documents.slice(i, i + batchSize);
-        await this.vectorStore.upsert(batch);
-        console.log(
-          `  ✅ Uploaded batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(documents.length / batchSize)}`,
-        );
-      }
-    }
-
-    console.log(`\n✨ Upload complete!`);
-    console.log(`   Uploaded: ${toUpload.length}`);
-    console.log(`   Deleted: ${toDelete.length}`);
-
-    return { uploaded: toUpload.length, deleted: toDelete.length };
-  }
-}

package/src/core/utils.ts

@@ -1,27 +0,0 @@
-import { createHash } from "crypto";
-
-export function computeContentHash(content: string): string {
-  return createHash("sha256").update(content).digest("hex").slice(0, 16);
-}
-
-export function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-export function batchArray<T>(array: T[], batchSize: number): T[][] {
-  const batches: T[][] = [];
-  for (let i = 0; i < array.length; i += batchSize) {
-    batches.push(array.slice(i, i + batchSize));
-  }
-  return batches;
-}
-
-export function extractFileName(filePath: string): string {
-  return filePath.split("/").pop() || filePath;
-}
-
-export function extractDirectory(filePath: string): string {
-  const parts = filePath.split("/");
-  parts.pop();
-  return parts.join("/");
-}

package/src/helpers/create-chunker.test.ts

@@ -1,31 +0,0 @@
-import { describe, it, expect, vi } from "vitest";
-import { createChunker } from "./create-chunker.js";
-
-describe("createChunker", () => {
-  it("should create a chunker with given options", () => {
-    const mockProcess = vi.fn().mockResolvedValue([]);
-
-    const chunker = createChunker({
-      name: "test-chunker",
-      patterns: ["**/*.txt"],
-      process: mockProcess,
-    });
-
-    expect(chunker.name).toBe("test-chunker");
-    expect(chunker.patterns).toEqual(["**/*.txt"]);
-    expect(chunker.chunk).toBeDefined();
-  });
-
-  it("should have canProcess method when provided", () => {
-    const mockCanProcess = vi.fn().mockResolvedValue(true);
-
-    const chunker = createChunker({
-      name: "test",
-      patterns: ["**/*.txt"],
-      process: vi.fn().mockResolvedValue([]),
-      canProcess: mockCanProcess,
-    });
-
-    expect(chunker.canProcess).toBeDefined();
-  });
-});

package/src/helpers/create-chunker.ts

@@ -1,40 +0,0 @@
-import { FileChunker, Chunk } from "../interfaces/index.js";
-
-export interface CreateChunkerOptions {
-  name: string;
-  patterns: string[];
-  process: (
-    content: string,
-    filePath: string,
-    commitHash: string,
-  ) => Promise<Chunk[]>;
-  canProcess?: (filePath: string, content?: string) => Promise<boolean>;
-}
-
-export function createChunker(options: CreateChunkerOptions): FileChunker {
-  return {
-    name: options.name,
-    patterns: options.patterns,
-
-    async chunk(filePath: string, commitHash: string): Promise<Chunk[]> {
-      const { readFile } = await import("fs/promises");
-      const content = await readFile(filePath, "utf-8");
-
-      if (options.canProcess) {
-        const canProcess = await options.canProcess(filePath, content);
-        if (!canProcess) {
-          return [];
-        }
-      }
-
-      return options.process(content, filePath, commitHash);
-    },
-
-    async canProcess(filePath: string, content?: string): Promise<boolean> {
-      if (options.canProcess) {
-        return options.canProcess(filePath, content);
-      }
-      return true;
-    },
-  };
-}

package/src/index.test.ts

@@ -1,33 +0,0 @@
-import { describe, it, expect } from "vitest";
-
-describe("@vivantel/rag-core", () => {
-  it("should export all public interfaces", async () => {
-    const module = await import("./index.js");
-
-    // Core
-    expect(module.GitTracker).toBeDefined();
-    expect(module.ChunkProcessor).toBeDefined();
-    expect(module.EmbedderProcessor).toBeDefined();
-    expect(module.Uploader).toBeDefined();
-    expect(module.Orchestrator).toBeDefined();
-
-    // Utils
-    expect(module.computeContentHash).toBeDefined();
-    expect(module.sleep).toBeDefined();
-    expect(module.batchArray).toBeDefined();
-    expect(module.extractFileName).toBeDefined();
-    expect(module.extractDirectory).toBeDefined();
-
-    // Strategies
-    expect(module.tokenStrategy).toBeDefined();
-    expect(module.markdownHeadersStrategy).toBeDefined();
-    expect(module.semanticStrategy).toBeDefined();
-    expect(module.wholeFileStrategy).toBeDefined();
-
-    // Helpers
-    expect(module.createChunker).toBeDefined();
-
-    // Config loader
-    expect(module.loadConfig).toBeDefined();
-  });
-});

package/src/index.ts

@@ -1,30 +0,0 @@
-// Interfaces
-export * from "./interfaces/index.js";
-
-// Core
-export { GitTracker } from "./core/git-tracker.js";
-export { ChunkProcessor } from "./core/chunk-processor.js";
-export { EmbedderProcessor } from "./core/embedder.js";
-export { Uploader } from "./core/uploader.js";
-export { Orchestrator, RAGPipelineConfig } from "./core/orchestrator.js";
-export {
-  computeContentHash,
-  sleep,
-  batchArray,
-  extractFileName,
-  extractDirectory,
-} from "./core/utils.js";
-
-// Strategies
-export {
-  tokenStrategy,
-  markdownHeadersStrategy,
-  semanticStrategy,
-  wholeFileStrategy,
-} from "./strategies/chunk/index.js";
-
-// Helpers
-export { createChunker } from "./helpers/create-chunker.js";
-
-// Config loader
-export { loadConfig } from "./config-loader.js";

package/src/interfaces/chunker.ts

@@ -1,59 +0,0 @@
-/**
- * Chunk interfaces - core building blocks for document processing
- */
-
-export interface Chunk {
-  /** The actual text content of the chunk */
-  content: string;
-
-  /** Metadata about this chunk (source file, type, etc.) */
-  metadata: Record<string, unknown>;
-
-  /** Original source file path */
-  sourceFile: string;
-
-  /** Git commit hash when this chunk was generated */
-  commitHash: string;
-
-  /** Optional unique hash of the content (for change detection) */
-  contentHash?: string;
-}
-
-export interface FileChunker {
-  /** Unique name of this chunker */
-  name: string;
-
-  /** Glob patterns this chunker handles */
-  patterns: string[];
-
-  /**
-   * Process a file and return chunks.
-   * Returns empty array if file should be skipped.
-   */
-  chunk(filePath: string, commitHash: string): Promise<Chunk[]>;
-
-  /**
-   * Optional: validate if this chunker can process the file
-   * (called before chunk() to filter early)
-   */
-  canProcess?(filePath: string, content?: string): Promise<boolean>;
-}
-
-export interface ChunkStrategy {
-  /** Strategy name */
-  name: string;
-
-  /** Split text into chunks according to strategy */
-  chunk(text: string, filePath?: string): Promise<Chunk[]>;
-
-  /** Optional: extract metadata without full chunking */
-  extractMetadata?(text: string, filePath?: string): Record<string, unknown>;
-}
-
-export interface ChunkTransformer {
-  /** Transformer name */
-  name: string;
-
-  /** Transform a chunk (return null to skip) */
-  transform(chunk: Chunk): Promise<Chunk | null>;
-}