@vivantel/rag-core 0.1.0 → 1.1.3

package/src/interfaces/embedder.ts

@@ -1,36 +0,0 @@
-/**
- * Embedding provider interfaces
- */
-
-import { Chunk } from "./chunker.js";
-
-export interface EmbeddingProvider {
-  /** Provider name (e.g., 'github-models', 'openai') */
-  readonly name: string;
-
-  /** Embedding vector dimensions */
-  readonly dimensions: number;
-
-  /** Maximum tokens per request (optional) */
-  readonly maxTokens?: number;
-
-  /** Convert text to embedding vector */
-  embed(text: string): Promise<number[]>;
-
-  /** Batch convert (optional, for performance) */
-  embedBatch?(texts: string[]): Promise<number[][]>;
-
-  /** Check if provider is available (e.g., valid API key) */
-  healthCheck?(): Promise<boolean>;
-}
-
-export interface EmbeddingConfig {
-  provider: EmbeddingProvider;
-  batchSize?: number;
-  rateLimitMs?: number;
-}
-
-export interface EmbeddedChunk extends Chunk {
-  embedding: number[];
-  embeddedAt: number;
-}

package/src/interfaces/index.test.ts

@@ -1,9 +0,0 @@
-import { describe, it, expect } from "vitest";
-
-describe("Interfaces", () => {
-  it("should export types correctly", async () => {
-    const module = await import("./index.js");
-
-    expect(module).toBeDefined();
-  });
-});

package/src/interfaces/index.ts

@@ -1,3 +0,0 @@
-export * from "./chunker.js";
-export * from "./embedder.js";
-export * from "./vector-store.js";

package/src/interfaces/vector-store.ts

@@ -1,71 +0,0 @@
-/**
- * Vector store interfaces
- */
-
-export interface VectorDocument {
-  /** Unique ID (optional, auto-generated if not provided) */
-  id?: string;
-
-  /** Original text content */
-  content: string;
-
-  /** Metadata for filtering */
-  metadata: Record<string, unknown>;
-
-  /** Embedding vector */
-  embedding: number[];
-
-  /** Source file path (for tracking updates) */
-  sourceFile: string;
-
-  /** Git commit hash (for change detection) */
-  commitHash: string;
-
-  /** Content hash (for change detection) */
-  contentHash: string;
-
-  /** Collection name (for multi-collection stores) */
-  collection?: string;
-}
-
-export interface VectorSearchResult {
-  id: string;
-  content: string;
-  metadata: Record<string, unknown>;
-  similarity: number;
-}
-
-export interface VectorStore {
-  /** Store name */
-  readonly name: string;
-
-  /** Initialize store (create tables, indexes, etc.) */
-  initialize(): Promise<void>;
-
-  /** Insert or update documents */
-  upsert(documents: VectorDocument[]): Promise<void>;
-
-  /** Delete documents by source file */
-  deleteBySourceFile(sourceFiles: string[]): Promise<void>;
-
-  /** Get current state (sourceFile → commitHash) for change detection */
-  getCurrentState(collection?: string): Promise<Map<string, string>>;
-
-  /** Search by embedding vector */
-  search(
-    queryEmbedding: number[],
-    topK: number,
-    collection?: string,
-  ): Promise<VectorSearchResult[]>;
-
-  /** Optional: delete entire collection */
-  deleteCollection?(collection: string): Promise<void>;
-
-  /** Optional: get store statistics */
-  getStats?(): Promise<{ documentCount: number; collections: string[] }>;
-}
-
-export interface VectorStoreConfig {
-  provider: VectorStore;
-  collection?: string;
-}

package/src/strategies/chunk/index.ts

@@ -1,4 +0,0 @@
-export { tokenStrategy } from "./token.js";
-export { markdownHeadersStrategy } from "./markdown-headers.js";
-export { semanticStrategy } from "./semantic.js";
-export { wholeFileStrategy } from "./whole-file.js";

package/src/strategies/chunk/markdown-headers.test.ts

@@ -1,37 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { markdownHeadersStrategy } from "./markdown-headers.js";
-
-describe("markdownHeadersStrategy", () => {
-  const strategy = markdownHeadersStrategy({ minChunkSize: 10 });
-
-  it("should have correct name", () => {
-    expect(strategy.name).toBe("markdown-headers");
-  });
-
-  it("should split by headers", async () => {
-    const text = `# Header 1
-Content for header 1.
-
-## Header 2
-Content for header 2.
-
-### Header 3
-Content for header 3.`;
-
-    const chunks = await strategy.chunk(text);
-
-    expect(chunks.length).toBeGreaterThan(0);
-
-    for (const chunk of chunks) {
-      expect(chunk.metadata.header).toBeDefined();
-      expect(chunk.metadata.header_level).toBeDefined();
-    }
-  });
-
-  it("should handle text without headers", async () => {
-    const text = "Plain text without any markdown headers.";
-    const chunks = await strategy.chunk(text);
-
-    expect(Array.isArray(chunks)).toBe(true);
-  });
-});

package/src/strategies/chunk/markdown-headers.ts

@@ -1,106 +0,0 @@
-import { ChunkStrategy, Chunk } from "../../interfaces/index.js";
-
-export interface MarkdownHeadersOptions {
-  minChunkSize?: number;
-  maxChunkSize?: number;
-}
-
-export function markdownHeadersStrategy(
-  options: MarkdownHeadersOptions = {},
-): ChunkStrategy {
-  const minChunkSize = options.minChunkSize ?? 100;
-  const maxChunkSize = options.maxChunkSize ?? 8000;
-
-  return {
-    name: "markdown-headers",
-
-    async chunk(text: string, filePath?: string): Promise<Chunk[]> {
-      const chunks: Chunk[] = [];
-      const lines = text.split("\n");
-
-      let currentChunk: string[] = [];
-      let currentHeader = "";
-      let currentHeaderLevel = 0;
-
-      for (const line of lines) {
-        const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
-
-        if (headerMatch) {
-          // Save previous chunk if not empty
-          if (currentChunk.length > 0) {
-            const content = currentChunk.join("\n").trim();
-            if (content.length >= minChunkSize) {
-              chunks.push({
-                content,
-                metadata: {
-                  strategy: this.name,
-                  header: currentHeader,
-                  header_level: currentHeaderLevel,
-                  source_file: filePath,
-                },
-                sourceFile: filePath || "unknown",
-                commitHash: "",
-              });
-            }
-          }
-
-          // Start new chunk
-          currentHeaderLevel = headerMatch[1].length;
-          currentHeader = headerMatch[2];
-          currentChunk = [line];
-        } else {
-          currentChunk.push(line);
-        }
-
-        // Prevent chunks from getting too large
-        const currentSize = currentChunk.join("\n").length;
-        if (currentSize > maxChunkSize && currentChunk.length > 10) {
-          const content = currentChunk.join("\n").trim();
-          chunks.push({
-            content,
-            metadata: {
-              strategy: this.name,
-              header: currentHeader,
-              header_level: currentHeaderLevel,
-              truncated: true,
-            },
-            sourceFile: filePath || "unknown",
-            commitHash: "",
-          });
-          currentChunk = [];
-        }
-      }
-
-      // Last chunk
-      if (currentChunk.length > 0) {
-        const content = currentChunk.join("\n").trim();
-        if (content.length >= minChunkSize) {
-          chunks.push({
-            content,
-            metadata: {
-              strategy: this.name,
-              header: currentHeader,
-              header_level: currentHeaderLevel,
-              source_file: filePath,
-              is_last: true,
-            },
-            sourceFile: filePath || "unknown",
-            commitHash: "",
-          });
-        }
-      }
-
-      return chunks;
-    },
-
-    extractMetadata(text: string, _filePath?: string): Record<string, unknown> {
-      const headerMatch = text.match(/^(#{1,6})\s+(.+)$/m);
-      return {
-        strategy: this.name,
-        has_headers: !!headerMatch,
-        first_header: headerMatch?.[2],
-        line_count: text.split("\n").length,
-      };
-    },
-  };
-}

package/src/strategies/chunk/semantic.test.ts

@@ -1,21 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { semanticStrategy } from "./semantic.js";
-
-describe("semanticStrategy", () => {
-  const strategy = semanticStrategy({ maxChars: 100, minChars: 10 });
-
-  it("should have correct name", () => {
-    expect(strategy.name).toBe("semantic");
-  });
-
-  it("should split by sentences", async () => {
-    const text = "First sentence. Second sentence! Third sentence? Fourth.";
-    const chunks = await strategy.chunk(text);
-
-    expect(Array.isArray(chunks)).toBe(true);
-
-    for (const chunk of chunks) {
-      expect(chunk.metadata.strategy).toBe("semantic");
-    }
-  });
-});

package/src/strategies/chunk/semantic.ts

@@ -1,80 +0,0 @@
-import { ChunkStrategy, Chunk } from "../../interfaces/index.js";
-
-export interface SemanticStrategyOptions {
-  maxChars?: number;
-  minChars?: number;
-}
-
-export function semanticStrategy(
-  options: SemanticStrategyOptions = {},
-): ChunkStrategy {
-  const maxChars = options.maxChars ?? 2000;
-  const minChars = options.minChars ?? 100;
-
-  return {
-    name: "semantic",
-
-    async chunk(text: string, filePath?: string): Promise<Chunk[]> {
-      const chunks: Chunk[] = [];
-
-      // Split by sentences (simple approach)
-      const sentences = text.split(/(?<=[.!?])\s+/);
-
-      let currentChunk: string[] = [];
-      let currentSize = 0;
-
-      for (const sentence of sentences) {
-        const sentenceSize = sentence.length;
-
-        if (currentSize + sentenceSize > maxChars && currentChunk.length > 0) {
-          const content = currentChunk.join(" ").trim();
-          if (content.length >= minChars) {
-            chunks.push({
-              content,
-              metadata: {
-                strategy: this.name,
-                sentence_count: currentChunk.length,
-                source_file: filePath,
-              },
-              sourceFile: filePath || "unknown",
-              commitHash: "",
-            });
-          }
-          currentChunk = [];
-          currentSize = 0;
-        }
-
-        currentChunk.push(sentence);
-        currentSize += sentenceSize;
-      }
-
-      // Last chunk
-      if (currentChunk.length > 0) {
-        const content = currentChunk.join(" ").trim();
-        if (content.length >= minChars) {
-          chunks.push({
-            content,
-            metadata: {
-              strategy: this.name,
-              sentence_count: currentChunk.length,
-              source_file: filePath,
-              is_last: true,
-            },
-            sourceFile: filePath || "unknown",
-            commitHash: "",
-          });
-        }
-      }
-
-      return chunks;
-    },
-
-    extractMetadata(text: string, _filePath?: string): Record<string, unknown> {
-      return {
-        strategy: this.name,
-        sentence_count: text.split(/[.!?]+/).length,
-        char_count: text.length,
-      };
-    },
-  };
-}

package/src/strategies/chunk/token.test.ts

@@ -1,41 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { tokenStrategy } from "./token.js";
-
-describe.skip("tokenStrategy", () => {
-  const strategy = tokenStrategy({ maxTokens: 50, overlap: 10 });
-
-  it("should have correct name", () => {
-    expect(strategy.name).toContain("token");
-    expect(typeof strategy.name).toBe("string");
-  });
-
-  it("should chunk text", async () => {
-    const text = "This is a test sentence. ".repeat(100);
-    const chunks = await strategy.chunk(text, "test.txt");
-
-    expect(Array.isArray(chunks)).toBe(true);
-    expect(chunks.length).toBeGreaterThan(0);
-
-    if (chunks.length > 0) {
-      expect(chunks[0].content).toBeDefined();
-      expect(typeof chunks[0].content).toBe("string");
-      expect(chunks[0].metadata).toBeDefined();
-      expect(chunks[0].metadata.strategy).toBeDefined();
-    }
-  });
-
-  it("should extract metadata", () => {
-    const text = "Test content";
-    const metadata = strategy.extractMetadata?.(text);
-
-    // extractMetadata is optional, so it might be undefined
-    if (metadata) {
-      expect(metadata.strategy).toBe(strategy.name);
-      expect(metadata.char_count).toBeDefined();
-      expect(metadata.estimated_tokens).toBeDefined();
-    } else {
-      // If extractMetadata is not implemented, just pass
-      expect(true).toBe(true);
-    }
-  });
-});

package/src/strategies/chunk/token.ts

@@ -1,72 +0,0 @@
-import { ChunkStrategy, Chunk } from "../../interfaces/index.js";
-
-export interface TokenStrategyOptions {
-  maxTokens?: number;
-  overlap?: number;
-}
-
-/**
- * Split text by approximate token count.
- * Simple implementation: ~4 chars per token for English.
- * For production, use a proper tokenizer (tiktoken, etc.)
- */
-export function tokenStrategy(
-  options: TokenStrategyOptions = {},
-): ChunkStrategy {
-  const maxTokens = options.maxTokens ?? 500;
-  const overlap = options.overlap ?? 50;
-  const charsPerToken = 4;
-  const maxChars = maxTokens * charsPerToken;
-  const overlapChars = overlap * charsPerToken;
-
-  return {
-    name: `token-${maxTokens}`,
-
-    async chunk(text: string, filePath?: string): Promise<Chunk[]> {
-      const chunks: Chunk[] = [];
-      let start = 0;
-
-      while (start < text.length) {
-        let end = Math.min(start + maxChars, text.length);
-
-        // Try to break at sentence boundary
-        if (end < text.length) {
-          const lastPeriod = text.lastIndexOf(".", end);
-          const lastNewline = text.lastIndexOf("\n", end);
-          const breakPoint = Math.max(lastPeriod, lastNewline);
-          if (breakPoint > start) {
-            end = breakPoint + 1;
-          }
-        }
-
-        const content = text.slice(start, end).trim();
-        if (content) {
-          chunks.push({
-            content,
-            metadata: {
-              strategy: this.name,
-              chunk_index: chunks.length,
-              source_file: filePath,
-              start_char: start,
-              end_char: end,
-            },
-            sourceFile: filePath || "unknown",
-            commitHash: "", // Will be filled by caller
-          });
-        }
-
-        start = end - overlapChars;
-      }
-
-      return chunks;
-    },
-
-    extractMetadata(text: string, _filePath?: string): Record<string, unknown> {
-      return {
-        strategy: this.name,
-        char_count: text.length,
-        estimated_tokens: Math.ceil(text.length / charsPerToken),
-      };
-    },
-  };
-}

package/src/strategies/chunk/whole-file.test.ts

@@ -1,24 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { wholeFileStrategy } from "./whole-file.js";
-
-describe("wholeFileStrategy", () => {
-  const strategy = wholeFileStrategy();
-
-  it("should have correct name", () => {
-    expect(strategy.name).toBe("whole-file");
-  });
-
-  it("should return single chunk", async () => {
-    const text = "Complete file content.";
-    const chunks = await strategy.chunk(text);
-
-    expect(chunks).toHaveLength(1);
-    expect(chunks[0].content).toBe(text);
-    expect(chunks[0].metadata.strategy).toBe("whole-file");
-  });
-
-  it("should return empty array for empty text", async () => {
-    const chunks = await strategy.chunk("");
-    expect(chunks).toHaveLength(0);
-  });
-});

package/src/strategies/chunk/whole-file.ts

@@ -1,35 +0,0 @@
-import { ChunkStrategy, Chunk } from "../../interfaces/index.js";
-
-export function wholeFileStrategy(): ChunkStrategy {
-  return {
-    name: "whole-file",
-
-    async chunk(text: string, filePath?: string): Promise<Chunk[]> {
-      if (!text || text.trim().length === 0) {
-        return [];
-      }
-
-      return [
-        {
-          content: text,
-          metadata: {
-            strategy: this.name,
-            source_file: filePath,
-            char_count: text.length,
-            line_count: text.split("\n").length,
-          },
-          sourceFile: filePath || "unknown",
-          commitHash: "",
-        },
-      ];
-    },
-
-    extractMetadata(text: string, _filePath?: string): Record<string, unknown> {
-      return {
-        strategy: this.name,
-        char_count: text.length,
-        line_count: text.split("\n").length,
-      };
-    },
-  };
-}

package/tsconfig.json

@@ -1,21 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "lib": ["ES2022"],
-    "outDir": "./dist",
-    "rootDir": "./src",
-    "declaration": true,
-    "declarationMap": true,
-    "sourceMap": true,
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "resolveJsonModule": true,
-    "types": ["node"]
-  },
-  "include": ["src/**/*"],
-  "exclude": ["node_modules", "dist", "**/*.test.ts"]
-}

package/typedoc.json

@@ -1,11 +0,0 @@
-{
-  "entryPoints": ["src/index.ts"],
-  "out": "docs/api",
-  "excludePrivate": true,
-  "excludeProtected": true,
-  "skipErrorChecking": true,
-  "validation": {
-    "invalidLink": false,
-    "notExported": false
-  }
-}

package/vitest.config.ts

@@ -1,19 +0,0 @@
-import { defineConfig } from 'vitest/config';
-
-export default defineConfig({
-  test: {
-    maxWorkers: 4,
-    isolate: false,
-
-    coverage: {
-      provider: 'v8',
-      reporter: ['text', 'json', 'html'],
-      include: ['src/**/*.ts'],
-      exclude: ['src/**/*.test.ts'],
-    },
-    exclude: ['node_modules', '.git'],
-    globals: true,
-    testTimeout: 10000,
-    environment: 'node',
-  },
-});