@vivantel/rag-core 0.1.0

package/dist/strategies/chunk/whole-file.js

@@ -0,0 +1,31 @@
+export function wholeFileStrategy() {
+    return {
+        name: "whole-file",
+        async chunk(text, filePath) {
+            if (!text || text.trim().length === 0) {
+                return [];
+            }
+            return [
+                {
+                    content: text,
+                    metadata: {
+                        strategy: this.name,
+                        source_file: filePath,
+                        char_count: text.length,
+                        line_count: text.split("\n").length,
+                    },
+                    sourceFile: filePath || "unknown",
+                    commitHash: "",
+                },
+            ];
+        },
+        extractMetadata(text, _filePath) {
+            return {
+                strategy: this.name,
+                char_count: text.length,
+                line_count: text.split("\n").length,
+            };
+        },
+    };
+}
+//# sourceMappingURL=whole-file.js.map

package/dist/strategies/chunk/whole-file.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"whole-file.js","sourceRoot":"","sources":["../../../src/strategies/chunk/whole-file.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,iBAAiB;IAC/B,OAAO;QACL,IAAI,EAAE,YAAY;QAElB,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,QAAiB;YACzC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtC,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,OAAO;gBACL;oBACE,OAAO,EAAE,IAAI;oBACb,QAAQ,EAAE;wBACR,QAAQ,EAAE,IAAI,CAAC,IAAI;wBACnB,WAAW,EAAE,QAAQ;wBACrB,UAAU,EAAE,IAAI,CAAC,MAAM;wBACvB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;qBACpC;oBACD,UAAU,EAAE,QAAQ,IAAI,SAAS;oBACjC,UAAU,EAAE,EAAE;iBACf;aACF,CAAC;QACJ,CAAC;QAED,eAAe,CAAC,IAAY,EAAE,SAAkB;YAC9C,OAAO;gBACL,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,UAAU,EAAE,IAAI,CAAC,MAAM;gBACvB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;aACpC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}

package/eslint.config.js

@@ -0,0 +1,25 @@
+import js from '@eslint/js';
+import tseslint from 'typescript-eslint';
+
+export default tseslint.config(
+  js.configs.recommended,
+  ...tseslint.configs.recommended,
+  {
+    ignores: [
+      'dist/**',
+      'node_modules/**',
+      'coverage/**',
+      '*.config.js',
+      '*.config.ts'
+    ]
+  },
+  {
+    files: ['src/**/*.ts'],
+    rules: {
+      '@typescript-eslint/no-explicit-any': 'warn',
+      '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
+      'no-console': 'off',
+      'no-undef': 'off'
+    }
+  }
+);

package/package.json

@@ -0,0 +1,102 @@
+{
+  "name": "@vivantel/rag-core",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Core RAG pipeline tools - universal chunking, embedding, vector store interfaces",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "rag-update": "./dist/bin/rag-update.js"
+  },
+  "scripts": {
+    "audit:fix": "npm audit fix",
+    "audit:report": "npm audit --json > npm-audit-report.json",
+    "audit": "npm audit",
+    "build:clean": "rm -rf dist && npm run build",
+    "build:verbose": "tsc --listEmittedFiles",
+    "build": "tsc && echo \"✅ Build completed successfully\"",
+    "check:full": "npm run check && npm run test:coverage && npm run type-coverage",
+    "check:quick": "npm run type-check && npm run lint",
+    "check": "npm run type-check && npm run lint && npm run format:check && npm run audit",
+    "clean:cache": "npm cache clean --force",
+    "clean:rebuild": "npm run clean && npm install && npm run build",
+    "clean": "rm -rf dist node_modules coverage docs",
+    "dev:debug": "tsx --inspect-brk src/index.ts",
+    "dev": "tsx watch src/index.ts",
+    "docs:generate": "typedoc",
+    "docs:open": "npm run docs:generate && open docs/index.html",
+    "docs:serve": "npm run docs:generate && npx http-server docs -p 8080",
+    "fix:aggressive": "npm run fix:all && npm run update:apply && npm run clean:rebuild",
+    "fix:all": "npm run fix && npm run audit:fix",
+    "fix": "npm run lint:fix && npm run format",
+    "format:check": "prettier --check \"src/**/*.ts\"",
+    "format": "prettier --write \"src/**/*.ts\"",
+    "help": "echo 'Available commands:' && grep -E '^\\s+\"[a-z]' package.json | sed 's/\"//g' | sed 's/://g' | column -t",
+    "info": "echo 'Node: $(node --version) | npm: $(npm --version)' && npm list --depth=0",
+    "lint:fix": "eslint src/ --fix",
+    "lint:strict": "eslint src/ --max-warnings 0",
+    "lint": "eslint src/",
+    "outdated:report": "npm outdated --json > npm-outdated-report.json",
+    "outdated": "npm outdated",
+    "postpublish": "echo \"✅ Published to npm\"",
+    "postversion": "git push && git push --tags",
+    "precommit": "npm run type-check && npm run format:check && npm run lint && npm audit",
+    "prepublishOnly": "npm run build && npm test",
+    "prepush": "npm run test:run && npm run audit && npm run format:check",
+    "release:major": "npm version major && npm publish",
+    "release:minor": "npm version minor && npm publish",
+    "release:patch": "npm version patch && npm publish",
+    "release:pre": "npm version prerelease && npm publish --tag next",
+    "status": "npm run outdated && npm run audit",
+    "test:coverage": "vitest --coverage --config vitest.config.ts",
+    "test:debug": "vitest --inspect-brk --config vitest.config.ts",
+    "test:run": "vitest run --config vitest.config.ts",
+    "test:update": "vitest -u --config vitest.config.ts",
+    "test:watch": "vitest watch --config vitest.config.ts",
+    "test": "vitest run --config vitest.config.ts",
+    "type-check:watch": "tsc --noEmit --watch",
+    "type-check": "tsc --noEmit",
+    "type-coverage:report": "type-coverage --detail",
+    "type-coverage": "type-coverage --at-least 90",
+    "update:apply": "npx npm-check-updates -u && npm install",
+    "update:check": "npx npm-check-updates",
+    "update": "npm update",
+    "version": "conventional-changelog -p angular -i CHANGELOG.md -s && git add CHANGELOG.md",
+    "watch:test": "vitest --watch --config vitest.config.ts",
+    "watch": "tsc --watch"
+  },
+  "keywords": [
+    "rag",
+    "embeddings",
+    "vector",
+    "git",
+    "pipeline"
+  ],
+  "author": "Vivantel",
+  "license": "MIT",
+  "dependencies": {
+    "commander": "^15.0.0",
+    "dotenv": "^17.4.2",
+    "glob": "^13.0.6",
+    "simple-git": "^3.25.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@types/node": "^25.9.1",
+    "eslint": "^10.4.1",
+    "prettier": "^3.8.3",
+    "tsx": "^4.11.0",
+    "typedoc": "^0.28.19",
+    "typedoc-plugin-markdown": "^4.11.0",
+    "typescript": "^6.0.3",
+    "typescript-eslint": "^8.60.0",
+    "vitest": "^4.1.7"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "vitest": {
+    "environment": "node",
+    "globals": false
+  }
+}

package/src/config-loader.ts

@@ -0,0 +1,21 @@
+import { pathToFileURL } from "url";
+import { RAGPipelineConfig } from "./core/orchestrator.js";
+
+export async function loadConfig(
+  configPath: string,
+): Promise<RAGPipelineConfig> {
+  // Clear cache for hot reload
+  delete require.cache[require.resolve(configPath)];
+
+  const configUrl = pathToFileURL(configPath).href;
+  const configModule = await import(configUrl);
+  const config = configModule.default;
+
+  if (!config.chunkers || !config.embedder || !config.vectorStore) {
+    throw new Error(
+      "Invalid config: missing chunkers, embedder, or vectorStore",
+    );
+  }
+
+  return config;
+}

package/src/core/chunk-processor.test.ts

@@ -0,0 +1,36 @@
+import { describe, it, expect, vi } from "vitest";
+import { ChunkProcessor } from "./chunk-processor.js";
+import { FileChunker } from "../interfaces/index.js";
+
+describe("ChunkProcessor", () => {
+  const mockChunker: FileChunker = {
+    name: "test",
+    patterns: ["**/*.txt"],
+    chunk: vi.fn().mockResolvedValue([
+      {
+        content: "test content",
+        metadata: { type: "test" },
+        sourceFile: "test.txt",
+        commitHash: "abc123",
+        contentHash: "hash123",
+      },
+    ]),
+  };
+
+  it("should be instantiable", () => {
+    const processor = new ChunkProcessor([mockChunker]);
+    expect(processor).toBeInstanceOf(ChunkProcessor);
+  });
+
+  it("should have processFile method", () => {
+    const processor = new ChunkProcessor([mockChunker]);
+    expect(processor.processFile).toBeDefined();
+    expect(typeof processor.processFile).toBe("function");
+  });
+
+  it("should have processFiles method", () => {
+    const processor = new ChunkProcessor([mockChunker]);
+    expect(processor.processFiles).toBeDefined();
+    expect(typeof processor.processFiles).toBe("function");
+  });
+});

package/src/core/chunk-processor.ts

@@ -0,0 +1,92 @@
+import { Chunk, FileChunker } from "../interfaces/index.js";
+import { createHash } from "crypto";
+
+function computeContentHash(content: string): string {
+  return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+
+export class ChunkProcessor {
+  private chunkers: Map<string, FileChunker>;
+
+  constructor(chunkers: FileChunker[]) {
+    this.chunkers = new Map(chunkers.map((c) => [c.name, c]));
+  }
+
+  async processFile(
+    filePath: string,
+    commitHash: string,
+    chunker: FileChunker,
+  ): Promise<Chunk[]> {
+    try {
+      const chunks = await chunker.chunk(filePath, commitHash);
+
+      for (const chunk of chunks) {
+        chunk.contentHash = computeContentHash(chunk.content);
+        chunk.sourceFile = filePath;
+        chunk.commitHash = commitHash;
+      }
+
+      return chunks;
+    } catch (error) {
+      console.error(`  ❌ Error processing ${filePath}: ${error}`);
+      return [];
+    }
+  }
+
+  async processFiles(
+    files: string[],
+    fileState: Map<string, { commitHash: string; chunker: FileChunker }>,
+  ): Promise<Chunk[]> {
+    const allChunks: Chunk[] = [];
+
+    for (let i = 0; i < files.length; i++) {
+      const filePath = files[i];
+      const info = fileState.get(filePath);
+
+      if (!info) {
+        console.log(`  ⚠️ No chunker for: ${filePath}`);
+        continue;
+      }
+
+      console.log(`  [${i + 1}/${files.length}] ${filePath}`);
+
+      const chunks = await this.processFile(
+        filePath,
+        info.commitHash,
+        info.chunker,
+      );
+
+      if (chunks.length > 0) {
+        allChunks.push(...chunks);
+        console.log(`    ✅ Generated ${chunks.length} chunk(s)`);
+      } else {
+        console.log(`    ⚠️ No chunks generated (skipped)`);
+      }
+    }
+
+    return allChunks;
+  }
+
+  async saveChunksLocal(chunks: Chunk[], outputFile: string): Promise<void> {
+    const { dirname } = await import("path");
+    const { mkdir, writeFile, readFile } = await import("fs/promises");
+
+    await mkdir(dirname(outputFile), { recursive: true });
+
+    let existing: Chunk[] = [];
+    try {
+      const content = await readFile(outputFile, "utf-8");
+      existing = JSON.parse(content);
+    } catch {
+      // File doesn't exist
+    }
+
+    const processedFiles = new Set(chunks.map((c) => c.sourceFile));
+    const filtered = existing.filter((c) => !processedFiles.has(c.sourceFile));
+
+    const allChunks = [...filtered, ...chunks];
+
+    await writeFile(outputFile, JSON.stringify(allChunks, null, 2));
+    console.log(`\n💾 Saved ${allChunks.length} chunks to ${outputFile}`);
+  }
+}

package/src/core/embedder.ts

@@ -0,0 +1,189 @@
+import {
+  EmbeddingProvider,
+  EmbeddedChunk,
+  Chunk,
+} from "../interfaces/index.js";
+import { readFile, writeFile, mkdir } from "fs/promises";
+import { dirname } from "path";
+import { createHash } from "crypto";
+
+function chunkContentHash(chunk: Chunk): string {
+  if (chunk.contentHash) return chunk.contentHash;
+  return createHash("sha256").update(chunk.content).digest("hex").slice(0, 16);
+}
+
+export class EmbedderProcessor {
+  private provider: EmbeddingProvider;
+  private rateLimitMs: number;
+  private batchSize: number;
+
+  constructor(
+    provider: EmbeddingProvider,
+    options: { rateLimitMs?: number; batchSize?: number } = {},
+  ) {
+    this.provider = provider;
+    this.rateLimitMs = options.rateLimitMs ?? 500;
+    this.batchSize = options.batchSize ?? 10;
+  }
+
+  private async sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  async embedChunk(chunk: Chunk): Promise<EmbeddedChunk> {
+    const embedding = await this.provider.embed(chunk.content);
+
+    return {
+      ...chunk,
+      embedding,
+      embeddedAt: Date.now() / 1000,
+    };
+  }
+
+  async embedBatch(chunks: Chunk[]): Promise<EmbeddedChunk[]> {
+    const results: EmbeddedChunk[] = [];
+
+    if (this.provider.embedBatch && chunks.length >= this.batchSize) {
+      const texts = chunks.map((c) => c.content);
+      const embeddings = await this.provider.embedBatch(texts);
+
+      for (let i = 0; i < chunks.length; i++) {
+        results.push({
+          ...chunks[i],
+          embedding: embeddings[i],
+          embeddedAt: Date.now() / 1000,
+        });
+      }
+    } else {
+      for (let i = 0; i < chunks.length; i++) {
+        const chunk = chunks[i];
+        const eventType =
+          (chunk.metadata.event_type as string) ||
+          (chunk.metadata.title as string) ||
+          chunk.sourceFile.split("/").pop() ||
+          "unknown";
+
+        console.log(`  [${i + 1}/${chunks.length}] ${eventType}`);
+
+        const embedded = await this.embedChunk(chunk);
+        results.push(embedded);
+
+        if (this.rateLimitMs > 0 && i < chunks.length - 1) {
+          await this.sleep(this.rateLimitMs);
+        }
+      }
+    }
+
+    return results;
+  }
+
+  async getChunksToEmbed(
+    chunksFile: string,
+    force: boolean = false,
+  ): Promise<{
+    chunksToEmbed: Chunk[];
+  }> {
+    let chunks: Chunk[];
+    try {
+      const content = await readFile(chunksFile, "utf-8");
+      chunks = JSON.parse(content);
+    } catch {
+      throw new Error(`Chunks file not found: ${chunksFile}`);
+    }
+
+    console.log(`📖 Loaded ${chunks.length} chunks from ${chunksFile}`);
+
+    if (force) {
+      console.log("  ⚠️ Force mode: embedding all chunks");
+      return { chunksToEmbed: chunks };
+    }
+
+    let existingEmbeddings: EmbeddedChunk[] = [];
+    const embeddingsFile = chunksFile.replace("chunks", "embeddings");
+    try {
+      const content = await readFile(embeddingsFile, "utf-8");
+      existingEmbeddings = JSON.parse(content);
+    } catch {
+      // No existing embeddings
+    }
+
+    const existingState = new Map<string, EmbeddedChunk>();
+    for (const emb of existingEmbeddings) {
+      const hash = emb.contentHash || chunkContentHash(emb);
+      existingState.set(hash, emb);
+    }
+
+    console.log(`📊 Existing embeddings: ${existingState.size} chunks`);
+
+    const chunksToEmbed: Chunk[] = [];
+    for (const chunk of chunks) {
+      const chunkHash = chunkContentHash(chunk);
+      if (!existingState.has(chunkHash)) {
+        chunksToEmbed.push(chunk);
+      }
+    }
+
+    return { chunksToEmbed };
+  }
+
+  async saveEmbeddings(
+    newEmbeddings: EmbeddedChunk[],
+    chunksFile: string,
+    force: boolean = false,
+  ): Promise<void> {
+    const embeddingsFile = chunksFile.replace("chunks", "embeddings");
+    await mkdir(dirname(embeddingsFile), { recursive: true });
+
+    const newByHash = new Map<string, EmbeddedChunk>();
+    for (const emb of newEmbeddings) {
+      const hash = emb.contentHash || chunkContentHash(emb);
+      newByHash.set(hash, emb);
+    }
+
+    let existing: EmbeddedChunk[] = [];
+    if (!force) {
+      try {
+        const content = await readFile(embeddingsFile, "utf-8");
+        existing = JSON.parse(content);
+      } catch {
+        // No existing embeddings
+      }
+    }
+
+    const final = force
+      ? []
+      : existing.filter((e) => {
+          const hash = e.contentHash || chunkContentHash(e);
+          return !newByHash.has(hash);
+        });
+
+    final.push(...newEmbeddings);
+
+    await writeFile(embeddingsFile, JSON.stringify(final, null, 2));
+    console.log(`\n💾 Saved ${final.length} embeddings to ${embeddingsFile}`);
+    console.log(
+      `   New: ${newEmbeddings.length}, Existing: ${final.length - newEmbeddings.length}`,
+    );
+  }
+
+  async run(
+    chunksFile: string,
+    force: boolean = false,
+  ): Promise<EmbeddedChunk[]> {
+    console.log("🔢 Starting incremental embedding generation...");
+
+    const { chunksToEmbed } = await this.getChunksToEmbed(chunksFile, force);
+
+    if (chunksToEmbed.length === 0) {
+      console.log("\n✨ No chunks need embedding.");
+      return [];
+    }
+
+    console.log(`\n📝 Need to embed ${chunksToEmbed.length} chunks`);
+
+    const newEmbeddings = await this.embedBatch(chunksToEmbed);
+    await this.saveEmbeddings(newEmbeddings, chunksFile, force);
+
+    return newEmbeddings;
+  }
+}

package/src/core/git-tracker.test.ts

@@ -0,0 +1,64 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { GitTracker } from "./git-tracker.js";
+import { FileChunker } from "../interfaces/index.js";
+import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "fs";
+import { join } from "path";
+import { tmpdir } from "os";
+import { execSync } from "child_process";
+
+describe("GitTracker", () => {
+  let testDir: string;
+  let originalCwd: string;
+
+  const mockChunker: FileChunker = {
+    name: "test",
+    patterns: ["**/*.txt", "**/*.yaml", "**/*.json"],
+    chunk: vi.fn().mockResolvedValue([]),
+  };
+
+  beforeEach(() => {
+    testDir = mkdtempSync(join(tmpdir(), "git-test-"));
+    originalCwd = process.cwd();
+    process.chdir(testDir);
+
+    mkdirSync(join(testDir, "src", "events"), { recursive: true });
+
+    writeFileSync(join(testDir, "test.txt"), "test content");
+    writeFileSync(
+      join(testDir, "src", "events", "booking.yaml"),
+      "event_type: BookingCreated",
+    );
+    writeFileSync(join(testDir, "config.json"), '{"key": "value"}');
+
+    execSync("git init", { stdio: "ignore" });
+    execSync('git config user.email "test@example.com"', { stdio: "ignore" });
+    execSync('git config user.name "Test"', { stdio: "ignore" });
+    execSync("git add .", { stdio: "ignore" });
+    execSync('git commit -m "initial"', { stdio: "ignore" });
+  });
+
+  afterEach(() => {
+    process.chdir(originalCwd);
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  it("should be instantiable", () => {
+    const tracker = new GitTracker([mockChunker]);
+    expect(tracker).toBeInstanceOf(GitTracker);
+  });
+
+  it("should getAllTrackedFiles", async () => {
+    const tracker = new GitTracker([mockChunker]);
+    const files = await tracker.getAllTrackedFiles();
+
+    expect(files.length).toBeGreaterThan(0);
+    expect(files.some((f) => f.includes("test.txt"))).toBe(true);
+  });
+
+  it("should getCurrentState", async () => {
+    const tracker = new GitTracker([mockChunker]);
+    const state = await tracker.getCurrentState();
+
+    expect(state.size).toBeGreaterThan(0);
+  });
+});