@vivantel/rag-core 1.1.2 → 2.0.0

package/src/strategies/chunk/markdown-headers.ts

@@ -1,106 +0,0 @@
-import { ChunkStrategy, Chunk } from "../../interfaces/index.js";
-
-export interface MarkdownHeadersOptions {
-  minChunkSize?: number;
-  maxChunkSize?: number;
-}
-
-export function markdownHeadersStrategy(
-  options: MarkdownHeadersOptions = {},
-): ChunkStrategy {
-  const minChunkSize = options.minChunkSize ?? 100;
-  const maxChunkSize = options.maxChunkSize ?? 8000;
-
-  return {
-    name: "markdown-headers",
-
-    async chunk(text: string, filePath?: string): Promise<Chunk[]> {
-      const chunks: Chunk[] = [];
-      const lines = text.split("\n");
-
-      let currentChunk: string[] = [];
-      let currentHeader = "";
-      let currentHeaderLevel = 0;
-
-      for (const line of lines) {
-        const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
-
-        if (headerMatch) {
-          // Save previous chunk if not empty
-          if (currentChunk.length > 0) {
-            const content = currentChunk.join("\n").trim();
-            if (content.length >= minChunkSize) {
-              chunks.push({
-                content,
-                metadata: {
-                  strategy: this.name,
-                  header: currentHeader,
-                  header_level: currentHeaderLevel,
-                  source_file: filePath,
-                },
-                sourceFile: filePath || "unknown",
-                commitHash: "",
-              });
-            }
-          }
-
-          // Start new chunk
-          currentHeaderLevel = headerMatch[1].length;
-          currentHeader = headerMatch[2];
-          currentChunk = [line];
-        } else {
-          currentChunk.push(line);
-        }
-
-        // Prevent chunks from getting too large
-        const currentSize = currentChunk.join("\n").length;
-        if (currentSize > maxChunkSize && currentChunk.length > 10) {
-          const content = currentChunk.join("\n").trim();
-          chunks.push({
-            content,
-            metadata: {
-              strategy: this.name,
-              header: currentHeader,
-              header_level: currentHeaderLevel,
-              truncated: true,
-            },
-            sourceFile: filePath || "unknown",
-            commitHash: "",
-          });
-          currentChunk = [];
-        }
-      }
-
-      // Last chunk
-      if (currentChunk.length > 0) {
-        const content = currentChunk.join("\n").trim();
-        if (content.length >= minChunkSize) {
-          chunks.push({
-            content,
-            metadata: {
-              strategy: this.name,
-              header: currentHeader,
-              header_level: currentHeaderLevel,
-              source_file: filePath,
-              is_last: true,
-            },
-            sourceFile: filePath || "unknown",
-            commitHash: "",
-          });
-        }
-      }
-
-      return chunks;
-    },
-
-    extractMetadata(text: string, _filePath?: string): Record<string, unknown> {
-      const headerMatch = text.match(/^(#{1,6})\s+(.+)$/m);
-      return {
-        strategy: this.name,
-        has_headers: !!headerMatch,
-        first_header: headerMatch?.[2],
-        line_count: text.split("\n").length,
-      };
-    },
-  };
-}

package/src/strategies/chunk/semantic.test.ts

@@ -1,21 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { semanticStrategy } from "./semantic.js";
-
-describe("semanticStrategy", () => {
-  const strategy = semanticStrategy({ maxChars: 100, minChars: 10 });
-
-  it("should have correct name", () => {
-    expect(strategy.name).toBe("semantic");
-  });
-
-  it("should split by sentences", async () => {
-    const text = "First sentence. Second sentence! Third sentence? Fourth.";
-    const chunks = await strategy.chunk(text);
-
-    expect(Array.isArray(chunks)).toBe(true);
-
-    for (const chunk of chunks) {
-      expect(chunk.metadata.strategy).toBe("semantic");
-    }
-  });
-});

package/src/strategies/chunk/semantic.ts

@@ -1,80 +0,0 @@
-import { ChunkStrategy, Chunk } from "../../interfaces/index.js";
-
-export interface SemanticStrategyOptions {
-  maxChars?: number;
-  minChars?: number;
-}
-
-export function semanticStrategy(
-  options: SemanticStrategyOptions = {},
-): ChunkStrategy {
-  const maxChars = options.maxChars ?? 2000;
-  const minChars = options.minChars ?? 100;
-
-  return {
-    name: "semantic",
-
-    async chunk(text: string, filePath?: string): Promise<Chunk[]> {
-      const chunks: Chunk[] = [];
-
-      // Split by sentences (simple approach)
-      const sentences = text.split(/(?<=[.!?])\s+/);
-
-      let currentChunk: string[] = [];
-      let currentSize = 0;
-
-      for (const sentence of sentences) {
-        const sentenceSize = sentence.length;
-
-        if (currentSize + sentenceSize > maxChars && currentChunk.length > 0) {
-          const content = currentChunk.join(" ").trim();
-          if (content.length >= minChars) {
-            chunks.push({
-              content,
-              metadata: {
-                strategy: this.name,
-                sentence_count: currentChunk.length,
-                source_file: filePath,
-              },
-              sourceFile: filePath || "unknown",
-              commitHash: "",
-            });
-          }
-          currentChunk = [];
-          currentSize = 0;
-        }
-
-        currentChunk.push(sentence);
-        currentSize += sentenceSize;
-      }
-
-      // Last chunk
-      if (currentChunk.length > 0) {
-        const content = currentChunk.join(" ").trim();
-        if (content.length >= minChars) {
-          chunks.push({
-            content,
-            metadata: {
-              strategy: this.name,
-              sentence_count: currentChunk.length,
-              source_file: filePath,
-              is_last: true,
-            },
-            sourceFile: filePath || "unknown",
-            commitHash: "",
-          });
-        }
-      }
-
-      return chunks;
-    },
-
-    extractMetadata(text: string, _filePath?: string): Record<string, unknown> {
-      return {
-        strategy: this.name,
-        sentence_count: text.split(/[.!?]+/).length,
-        char_count: text.length,
-      };
-    },
-  };
-}

package/src/strategies/chunk/token.test.ts

@@ -1,41 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { tokenStrategy } from "./token.js";
-
-describe.skip("tokenStrategy", () => {
-  const strategy = tokenStrategy({ maxTokens: 50, overlap: 10 });
-
-  it("should have correct name", () => {
-    expect(strategy.name).toContain("token");
-    expect(typeof strategy.name).toBe("string");
-  });
-
-  it("should chunk text", async () => {
-    const text = "This is a test sentence. ".repeat(100);
-    const chunks = await strategy.chunk(text, "test.txt");
-
-    expect(Array.isArray(chunks)).toBe(true);
-    expect(chunks.length).toBeGreaterThan(0);
-
-    if (chunks.length > 0) {
-      expect(chunks[0].content).toBeDefined();
-      expect(typeof chunks[0].content).toBe("string");
-      expect(chunks[0].metadata).toBeDefined();
-      expect(chunks[0].metadata.strategy).toBeDefined();
-    }
-  });
-
-  it("should extract metadata", () => {
-    const text = "Test content";
-    const metadata = strategy.extractMetadata?.(text);
-
-    // extractMetadata is optional, so it might be undefined
-    if (metadata) {
-      expect(metadata.strategy).toBe(strategy.name);
-      expect(metadata.char_count).toBeDefined();
-      expect(metadata.estimated_tokens).toBeDefined();
-    } else {
-      // If extractMetadata is not implemented, just pass
-      expect(true).toBe(true);
-    }
-  });
-});

package/src/strategies/chunk/token.ts

@@ -1,72 +0,0 @@
-import { ChunkStrategy, Chunk } from "../../interfaces/index.js";
-
-export interface TokenStrategyOptions {
-  maxTokens?: number;
-  overlap?: number;
-}
-
-/**
- * Split text by approximate token count.
- * Simple implementation: ~4 chars per token for English.
- * For production, use a proper tokenizer (tiktoken, etc.)
- */
-export function tokenStrategy(
-  options: TokenStrategyOptions = {},
-): ChunkStrategy {
-  const maxTokens = options.maxTokens ?? 500;
-  const overlap = options.overlap ?? 50;
-  const charsPerToken = 4;
-  const maxChars = maxTokens * charsPerToken;
-  const overlapChars = overlap * charsPerToken;
-
-  return {
-    name: `token-${maxTokens}`,
-
-    async chunk(text: string, filePath?: string): Promise<Chunk[]> {
-      const chunks: Chunk[] = [];
-      let start = 0;
-
-      while (start < text.length) {
-        let end = Math.min(start + maxChars, text.length);
-
-        // Try to break at sentence boundary
-        if (end < text.length) {
-          const lastPeriod = text.lastIndexOf(".", end);
-          const lastNewline = text.lastIndexOf("\n", end);
-          const breakPoint = Math.max(lastPeriod, lastNewline);
-          if (breakPoint > start) {
-            end = breakPoint + 1;
-          }
-        }
-
-        const content = text.slice(start, end).trim();
-        if (content) {
-          chunks.push({
-            content,
-            metadata: {
-              strategy: this.name,
-              chunk_index: chunks.length,
-              source_file: filePath,
-              start_char: start,
-              end_char: end,
-            },
-            sourceFile: filePath || "unknown",
-            commitHash: "", // Will be filled by caller
-          });
-        }
-
-        start = end - overlapChars;
-      }
-
-      return chunks;
-    },
-
-    extractMetadata(text: string, _filePath?: string): Record<string, unknown> {
-      return {
-        strategy: this.name,
-        char_count: text.length,
-        estimated_tokens: Math.ceil(text.length / charsPerToken),
-      };
-    },
-  };
-}

package/src/strategies/chunk/whole-file.test.ts

@@ -1,24 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { wholeFileStrategy } from "./whole-file.js";
-
-describe("wholeFileStrategy", () => {
-  const strategy = wholeFileStrategy();
-
-  it("should have correct name", () => {
-    expect(strategy.name).toBe("whole-file");
-  });
-
-  it("should return single chunk", async () => {
-    const text = "Complete file content.";
-    const chunks = await strategy.chunk(text);
-
-    expect(chunks).toHaveLength(1);
-    expect(chunks[0].content).toBe(text);
-    expect(chunks[0].metadata.strategy).toBe("whole-file");
-  });
-
-  it("should return empty array for empty text", async () => {
-    const chunks = await strategy.chunk("");
-    expect(chunks).toHaveLength(0);
-  });
-});

package/src/strategies/chunk/whole-file.ts

@@ -1,35 +0,0 @@
-import { ChunkStrategy, Chunk } from "../../interfaces/index.js";
-
-export function wholeFileStrategy(): ChunkStrategy {
-  return {
-    name: "whole-file",
-
-    async chunk(text: string, filePath?: string): Promise<Chunk[]> {
-      if (!text || text.trim().length === 0) {
-        return [];
-      }
-
-      return [
-        {
-          content: text,
-          metadata: {
-            strategy: this.name,
-            source_file: filePath,
-            char_count: text.length,
-            line_count: text.split("\n").length,
-          },
-          sourceFile: filePath || "unknown",
-          commitHash: "",
-        },
-      ];
-    },
-
-    extractMetadata(text: string, _filePath?: string): Record<string, unknown> {
-      return {
-        strategy: this.name,
-        char_count: text.length,
-        line_count: text.split("\n").length,
-      };
-    },
-  };
-}

package/tsconfig.json

@@ -1,21 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "lib": ["ES2022"],
-    "outDir": "./dist",
-    "rootDir": "./src",
-    "declaration": true,
-    "declarationMap": true,
-    "sourceMap": true,
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "resolveJsonModule": true,
-    "types": ["node"]
-  },
-  "include": ["src/**/*"],
-  "exclude": ["node_modules", "dist", "**/*.test.ts"]
-}

package/typedoc.json

@@ -1,11 +0,0 @@
-{
-  "entryPoints": ["src/index.ts"],
-  "out": "docs/api",
-  "excludePrivate": true,
-  "excludeProtected": true,
-  "skipErrorChecking": true,
-  "validation": {
-    "invalidLink": false,
-    "notExported": false
-  }
-}

package/vitest.config.ts

@@ -1,19 +0,0 @@
-import { defineConfig } from 'vitest/config';
-
-export default defineConfig({
-  test: {
-    maxWorkers: 4,
-    isolate: false,
-
-    coverage: {
-      provider: 'v8',
-      reporter: ['text', 'json', 'html'],
-      include: ['src/**/*.ts'],
-      exclude: ['src/**/*.test.ts'],
-    },
-    exclude: ['node_modules', '.git'],
-    globals: true,
-    testTimeout: 10000,
-    environment: 'node',
-  },
-});