@cue-dev/retrieval-core 0.1.3

package/test/chunking.config.test.ts

@@ -0,0 +1,84 @@
+import { describe, expect, it } from "vitest";
+import { DEFAULT_RETRIEVAL_CHUNKING_CONFIG, mergeRetrievalChunkingConfig } from "../src/index.js";
+
+describe("retrieval-core chunking config validation", () => {
+  it("uses wider default chunk windows for better snippet completeness", () => {
+    expect(DEFAULT_RETRIEVAL_CHUNKING_CONFIG.strategy).toBe("language_aware");
+    expect(DEFAULT_RETRIEVAL_CHUNKING_CONFIG.fallback_strategy).toBe("sliding");
+    expect(DEFAULT_RETRIEVAL_CHUNKING_CONFIG.target_chunk_tokens).toBe(420);
+    expect(DEFAULT_RETRIEVAL_CHUNKING_CONFIG.chunk_overlap_tokens).toBe(90);
+    expect(DEFAULT_RETRIEVAL_CHUNKING_CONFIG.budget_tokenizer).toBe("ranking");
+    expect(DEFAULT_RETRIEVAL_CHUNKING_CONFIG.boundary_strictness).toBe("semantic_js_ts");
+    expect(DEFAULT_RETRIEVAL_CHUNKING_CONFIG.enabled_languages).toEqual([
+      "typescript",
+      "javascript",
+      "python",
+      "go",
+      "rust",
+      "java"
+    ]);
+    expect(DEFAULT_RETRIEVAL_CHUNKING_CONFIG.recursive_semantic_chunking_enabled).toBe(true);
+    expect(DEFAULT_RETRIEVAL_CHUNKING_CONFIG.embedding_context_prefix_enabled).toBe(true);
+  });
+
+  it("accepts valid chunk window overrides", () => {
+    const merged = mergeRetrievalChunkingConfig(DEFAULT_RETRIEVAL_CHUNKING_CONFIG, {
+      target_chunk_tokens: 512,
+      chunk_overlap_tokens: 128,
+      budget_tokenizer: "lightweight",
+      boundary_strictness: "semantic_js_ts",
+      recursive_semantic_chunking_enabled: true,
+      semantic_merge_gap_lines: 8,
+      semantic_merge_max_span_lines: 260,
+      comment_forward_absorb_enabled: false,
+      embedding_context_prefix_enabled: false
+    });
+
+    expect(merged.target_chunk_tokens).toBe(512);
+    expect(merged.chunk_overlap_tokens).toBe(128);
+    expect(merged.budget_tokenizer).toBe("lightweight");
+    expect(merged.boundary_strictness).toBe("semantic_js_ts");
+    expect(merged.recursive_semantic_chunking_enabled).toBe(true);
+    expect(merged.semantic_merge_gap_lines).toBe(8);
+    expect(merged.semantic_merge_max_span_lines).toBe(260);
+    expect(merged.comment_forward_absorb_enabled).toBe(false);
+    expect(merged.embedding_context_prefix_enabled).toBe(false);
+  });
+
+  it("rejects overlap values that are not less than target chunk tokens", () => {
+    expect(() =>
+      mergeRetrievalChunkingConfig(DEFAULT_RETRIEVAL_CHUNKING_CONFIG, {
+        target_chunk_tokens: 300,
+        chunk_overlap_tokens: 300
+      })
+    ).toThrow(/chunk_overlap_tokens must be less than target_chunk_tokens/);
+  });
+
+  it("rejects invalid budget tokenizer and boundary strictness values", () => {
+    expect(() =>
+      mergeRetrievalChunkingConfig(DEFAULT_RETRIEVAL_CHUNKING_CONFIG, {
+        budget_tokenizer: "invalid" as "ranking"
+      })
+    ).toThrow(/budget_tokenizer must be ranking\|lightweight/);
+
+    expect(() =>
+      mergeRetrievalChunkingConfig(DEFAULT_RETRIEVAL_CHUNKING_CONFIG, {
+        boundary_strictness: "invalid" as "legacy"
+      })
+    ).toThrow(/boundary_strictness must be legacy\|semantic_js_ts/);
+  });
+
+  it("rejects invalid semantic merge settings", () => {
+    expect(() =>
+      mergeRetrievalChunkingConfig(DEFAULT_RETRIEVAL_CHUNKING_CONFIG, {
+        semantic_merge_gap_lines: -1
+      })
+    ).toThrow(/semantic_merge_gap_lines must be a non-negative integer/);
+
+    expect(() =>
+      mergeRetrievalChunkingConfig(DEFAULT_RETRIEVAL_CHUNKING_CONFIG, {
+        semantic_merge_max_span_lines: 0
+      })
+    ).toThrow(/semantic_merge_max_span_lines must be a positive integer/);
+  });
+});