@rce-mcp/retrieval-core 0.1.2 → 0.1.3

package/src/index.ts

@@ -105,6 +105,15 @@ export interface RetrievalRerankConfig {
   max_chunks_per_path_file_lookup: number;
   same_directory_penalty: number;
   same_extension_penalty: number;
+  merge_overlapping_chunks_enabled: boolean;
+  merge_gap_lines: number;
+  merge_max_span_lines: number;
+  smart_cutoff_enabled: boolean;
+  smart_cutoff_min_k: number;
+  smart_cutoff_max_k: number;
+  smart_cutoff_min_score: number;
+  smart_cutoff_top_ratio: number;
+  smart_cutoff_delta_abs: number;
 }
 
 export interface RetrievalScoringConfig {
@@ -133,8 +142,33 @@ export interface RetrievalChunkingConfig {
   fallback_strategy: "sliding";
   target_chunk_tokens: number;
   chunk_overlap_tokens: number;
+  budget_tokenizer: "ranking" | "lightweight";
+  boundary_strictness: "legacy" | "semantic_js_ts";
   parse_timeout_ms: number;
   enabled_languages: string[];
+  recursive_semantic_chunking_enabled: boolean;
+  semantic_merge_gap_lines: number;
+  semantic_merge_max_span_lines: number;
+  comment_forward_absorb_enabled: boolean;
+  embedding_context_prefix_enabled: boolean;
+}
+
+export interface RetrievalContextPackingConfig {
+  enabled: boolean;
+  max_spans_per_result: number;
+  max_gap_lines: number;
+  max_snippet_chars: number;
+  enhancer_snippet_char_limit: number;
+}
+
+export interface RetrievalSnippetIntegrityConfig {
+  enabled: boolean;
+  target_languages: string[];
+  max_contiguous_gap_lines: number;
+  marker_template_version: "v1";
+  repair_enabled: boolean;
+  repair_max_envelope_lines: number;
+  repair_max_snippet_chars: number;
 }
 
 export type RetrievalScoringConfigInput = Partial<{
@@ -150,8 +184,25 @@ export type RetrievalChunkingConfigInput = Partial<{
   fallback_strategy: "sliding";
   target_chunk_tokens: number;
   chunk_overlap_tokens: number;
+  budget_tokenizer: "ranking" | "lightweight";
+  boundary_strictness: "legacy" | "semantic_js_ts";
   parse_timeout_ms: number;
   enabled_languages: string[];
+  recursive_semantic_chunking_enabled: boolean;
+  semantic_merge_gap_lines: number;
+  semantic_merge_max_span_lines: number;
+  comment_forward_absorb_enabled: boolean;
+  embedding_context_prefix_enabled: boolean;
+}>;
+export type RetrievalContextPackingConfigInput = Partial<RetrievalContextPackingConfig>;
+export type RetrievalSnippetIntegrityConfigInput = Partial<{
+  enabled: boolean;
+  target_languages: string[];
+  max_contiguous_gap_lines: number;
+  marker_template_version: "v1";
+  repair_enabled: boolean;
+  repair_max_envelope_lines: number;
+  repair_max_snippet_chars: number;
 }>;
 
 export const BASELINE_RETRIEVAL_SCORING_CONFIG: RetrievalScoringConfig = {
@@ -196,7 +247,16 @@ export const BASELINE_RETRIEVAL_SCORING_CONFIG: RetrievalScoringConfig = {
     max_chunks_per_path_default: 2,
     max_chunks_per_path_file_lookup: 1,
     same_directory_penalty: 0,
-    same_extension_penalty: 0
+    same_extension_penalty: 0,
+    merge_overlapping_chunks_enabled: true,
+    merge_gap_lines: 6,
+    merge_max_span_lines: 220,
+    smart_cutoff_enabled: false,
+    smart_cutoff_min_k: 2,
+    smart_cutoff_max_k: 8,
+    smart_cutoff_min_score: 0.25,
+    smart_cutoff_top_ratio: 0.5,
+    smart_cutoff_delta_abs: 0.25
   }
 };
 
@@ -248,7 +308,16 @@ export const CONSERVATIVE_RETRIEVAL_SCORING_CONFIG: RetrievalScoringConfig = {
     max_chunks_per_path_default: 2,
     max_chunks_per_path_file_lookup: 1,
     same_directory_penalty: 0,
-    same_extension_penalty: 0
+    same_extension_penalty: 0,
+    merge_overlapping_chunks_enabled: true,
+    merge_gap_lines: 6,
+    merge_max_span_lines: 220,
+    smart_cutoff_enabled: false,
+    smart_cutoff_min_k: 2,
+    smart_cutoff_max_k: 8,
+    smart_cutoff_min_score: 0.25,
+    smart_cutoff_top_ratio: 0.5,
+    smart_cutoff_delta_abs: 0.25
   }
 };
 
@@ -270,8 +339,33 @@ export const DEFAULT_RETRIEVAL_CHUNKING_CONFIG: RetrievalChunkingConfig = {
   fallback_strategy: "sliding",
   target_chunk_tokens: DEFAULT_TARGET_CHUNK_TOKENS,
   chunk_overlap_tokens: DEFAULT_CHUNK_OVERLAP_TOKENS,
+  budget_tokenizer: "ranking",
+  boundary_strictness: "legacy",
   parse_timeout_ms: 80,
-  enabled_languages: ["typescript", "javascript", "python", "go"]
+  enabled_languages: ["typescript", "javascript", "python", "go"],
+  recursive_semantic_chunking_enabled: false,
+  semantic_merge_gap_lines: 6,
+  semantic_merge_max_span_lines: 220,
+  comment_forward_absorb_enabled: true,
+  embedding_context_prefix_enabled: true
+};
+
+export const DEFAULT_RETRIEVAL_CONTEXT_PACKING_CONFIG: RetrievalContextPackingConfig = {
+  enabled: false,
+  max_spans_per_result: 3,
+  max_gap_lines: 120,
+  max_snippet_chars: 3_200,
+  enhancer_snippet_char_limit: 2_200
+};
+
+export const DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG: RetrievalSnippetIntegrityConfig = {
+  enabled: false,
+  target_languages: ["typescript", "tsx", "javascript", "jsx"],
+  max_contiguous_gap_lines: 6,
+  marker_template_version: "v1",
+  repair_enabled: false,
+  repair_max_envelope_lines: 260,
+  repair_max_snippet_chars: 3_600
 };
 
 const BUILTIN_RETRIEVAL_SCORING_PROFILES = {
@@ -337,6 +431,36 @@ function validateScoringConfig(config: RetrievalScoringConfig): void {
   if (rerank.same_extension_penalty < 0) {
     throw new Error("invalid retrieval scoring config: rerank.same_extension_penalty must be >= 0");
   }
+  if (typeof rerank.merge_overlapping_chunks_enabled !== "boolean") {
+    throw new Error("invalid retrieval scoring config: rerank.merge_overlapping_chunks_enabled must be boolean");
+  }
+  if (!Number.isInteger(rerank.merge_gap_lines) || rerank.merge_gap_lines < 0) {
+    throw new Error("invalid retrieval scoring config: rerank.merge_gap_lines must be an integer >= 0");
+  }
+  if (!Number.isInteger(rerank.merge_max_span_lines) || rerank.merge_max_span_lines <= 0) {
+    throw new Error("invalid retrieval scoring config: rerank.merge_max_span_lines must be a positive integer");
+  }
+  if (typeof rerank.smart_cutoff_enabled !== "boolean") {
+    throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_enabled must be boolean");
+  }
+  if (!Number.isInteger(rerank.smart_cutoff_min_k) || rerank.smart_cutoff_min_k <= 0) {
+    throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_min_k must be a positive integer");
+  }
+  if (!Number.isInteger(rerank.smart_cutoff_max_k) || rerank.smart_cutoff_max_k <= 0) {
+    throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_max_k must be a positive integer");
+  }
+  if (rerank.smart_cutoff_max_k < rerank.smart_cutoff_min_k) {
+    throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_max_k must be >= smart_cutoff_min_k");
+  }
+  assertFiniteNumber(rerank.smart_cutoff_min_score, "rerank.smart_cutoff_min_score");
+  assertFiniteNumber(rerank.smart_cutoff_top_ratio, "rerank.smart_cutoff_top_ratio");
+  assertFiniteNumber(rerank.smart_cutoff_delta_abs, "rerank.smart_cutoff_delta_abs");
+  if (rerank.smart_cutoff_top_ratio <= 0 || rerank.smart_cutoff_top_ratio > 1) {
+    throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_top_ratio must be in (0, 1]");
+  }
+  if (rerank.smart_cutoff_delta_abs < 0) {
+    throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_delta_abs must be >= 0");
+  }
 }
 
 export function resolveRetrievalScoringProfile(profile_id: string | undefined): {
@@ -466,6 +590,12 @@ function validateChunkingConfig(config: RetrievalChunkingConfig): void {
   if (!Number.isInteger(config.parse_timeout_ms) || config.parse_timeout_ms <= 0) {
     throw new Error("invalid retrieval chunking config: parse_timeout_ms must be a positive integer");
   }
+  if (config.budget_tokenizer !== "ranking" && config.budget_tokenizer !== "lightweight") {
+    throw new Error("invalid retrieval chunking config: budget_tokenizer must be ranking|lightweight");
+  }
+  if (config.boundary_strictness !== "legacy" && config.boundary_strictness !== "semantic_js_ts") {
+    throw new Error("invalid retrieval chunking config: boundary_strictness must be legacy|semantic_js_ts");
+  }
   if (!Array.isArray(config.enabled_languages) || config.enabled_languages.length === 0) {
     throw new Error("invalid retrieval chunking config: enabled_languages must include at least one language");
   }
@@ -474,6 +604,21 @@ function validateChunkingConfig(config: RetrievalChunkingConfig): void {
       throw new Error("invalid retrieval chunking config: enabled_languages must contain non-empty strings");
     }
   }
+  if (typeof config.recursive_semantic_chunking_enabled !== "boolean") {
+    throw new Error("invalid retrieval chunking config: recursive_semantic_chunking_enabled must be boolean");
+  }
+  if (!Number.isInteger(config.semantic_merge_gap_lines) || config.semantic_merge_gap_lines < 0) {
+    throw new Error("invalid retrieval chunking config: semantic_merge_gap_lines must be a non-negative integer");
+  }
+  if (!Number.isInteger(config.semantic_merge_max_span_lines) || config.semantic_merge_max_span_lines <= 0) {
+    throw new Error("invalid retrieval chunking config: semantic_merge_max_span_lines must be a positive integer");
+  }
+  if (typeof config.comment_forward_absorb_enabled !== "boolean") {
+    throw new Error("invalid retrieval chunking config: comment_forward_absorb_enabled must be boolean");
+  }
+  if (typeof config.embedding_context_prefix_enabled !== "boolean") {
+    throw new Error("invalid retrieval chunking config: embedding_context_prefix_enabled must be boolean");
+  }
 }
 
 export function mergeRetrievalChunkingConfig(
@@ -489,6 +634,110 @@ export function mergeRetrievalChunkingConfig(
   return next;
 }
 
+function validateContextPackingConfig(config: RetrievalContextPackingConfig): void {
+  if (typeof config.enabled !== "boolean") {
+    throw new Error("invalid retrieval context packing config: enabled must be boolean");
+  }
+  if (!Number.isInteger(config.max_spans_per_result) || config.max_spans_per_result <= 0) {
+    throw new Error("invalid retrieval context packing config: max_spans_per_result must be a positive integer");
+  }
+  if (!Number.isInteger(config.max_gap_lines) || config.max_gap_lines < 0) {
+    throw new Error("invalid retrieval context packing config: max_gap_lines must be a non-negative integer");
+  }
+  if (!Number.isInteger(config.max_snippet_chars) || config.max_snippet_chars <= 0) {
+    throw new Error("invalid retrieval context packing config: max_snippet_chars must be a positive integer");
+  }
+  if (!Number.isInteger(config.enhancer_snippet_char_limit) || config.enhancer_snippet_char_limit <= 0) {
+    throw new Error("invalid retrieval context packing config: enhancer_snippet_char_limit must be a positive integer");
+  }
+}
+
+export function mergeRetrievalContextPackingConfig(
+  base: RetrievalContextPackingConfig,
+  overrides?: RetrievalContextPackingConfigInput
+): RetrievalContextPackingConfig {
+  const next: RetrievalContextPackingConfig = {
+    ...base,
+    ...(overrides ?? {})
+  };
+  validateContextPackingConfig(next);
+  return next;
+}
+
+function normalizeSnippetIntegrityLanguage(value: string): string | undefined {
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "typescript" || normalized === "ts" || normalized === "mts" || normalized === "cts") {
+    return "typescript";
+  }
+  if (normalized === "tsx") {
+    return "tsx";
+  }
+  if (normalized === "javascript" || normalized === "js" || normalized === "mjs" || normalized === "cjs") {
+    return "javascript";
+  }
+  if (normalized === "jsx") {
+    return "jsx";
+  }
+  return undefined;
+}
+
+function normalizeSnippetIntegrityLanguageList(value: string[]): string[] {
+  const deduped = new Set<string>();
+  for (const language of value) {
+    const raw = language.trim().toLowerCase();
+    if (raw.length === 0) {
+      continue;
+    }
+    deduped.add(normalizeSnippetIntegrityLanguage(raw) ?? raw);
+  }
+  return [...deduped];
+}
+
+function validateSnippetIntegrityConfig(config: RetrievalSnippetIntegrityConfig): void {
+  if (typeof config.enabled !== "boolean") {
+    throw new Error("invalid retrieval snippet integrity config: enabled must be boolean");
+  }
+  if (!Array.isArray(config.target_languages) || config.target_languages.length === 0) {
+    throw new Error("invalid retrieval snippet integrity config: target_languages must include at least one language");
+  }
+  for (const language of config.target_languages) {
+    if (typeof language !== "string" || language.trim().length === 0) {
+      throw new Error("invalid retrieval snippet integrity config: target_languages must contain non-empty strings");
+    }
+    if (!normalizeSnippetIntegrityLanguage(language)) {
+      throw new Error("invalid retrieval snippet integrity config: unsupported target language");
+    }
+  }
+  if (!Number.isInteger(config.max_contiguous_gap_lines) || config.max_contiguous_gap_lines < 0) {
+    throw new Error("invalid retrieval snippet integrity config: max_contiguous_gap_lines must be a non-negative integer");
+  }
+  if (config.marker_template_version !== "v1") {
+    throw new Error("invalid retrieval snippet integrity config: marker_template_version must be v1");
+  }
+  if (typeof config.repair_enabled !== "boolean") {
+    throw new Error("invalid retrieval snippet integrity config: repair_enabled must be boolean");
+  }
+  if (!Number.isInteger(config.repair_max_envelope_lines) || config.repair_max_envelope_lines <= 0) {
+    throw new Error("invalid retrieval snippet integrity config: repair_max_envelope_lines must be a positive integer");
+  }
+  if (!Number.isInteger(config.repair_max_snippet_chars) || config.repair_max_snippet_chars <= 0) {
+    throw new Error("invalid retrieval snippet integrity config: repair_max_snippet_chars must be a positive integer");
+  }
+}
+
+export function mergeRetrievalSnippetIntegrityConfig(
+  base: RetrievalSnippetIntegrityConfig,
+  overrides?: RetrievalSnippetIntegrityConfigInput
+): RetrievalSnippetIntegrityConfig {
+  const next: RetrievalSnippetIntegrityConfig = {
+    ...base,
+    ...(overrides ?? {}),
+    target_languages: normalizeSnippetIntegrityLanguageList(overrides?.target_languages ?? base.target_languages)
+  };
+  validateSnippetIntegrityConfig(next);
+  return next;
+}
+
 function stableSerialize(value: unknown): string {
   if (Array.isArray(value)) {
     return `[${value.map((entry) => stableSerialize(entry)).join(",")}]`;
@@ -621,6 +870,8 @@ export interface RetrievalCoreOptions {
   enhancerConfig?: RetrievalEnhancerConfigInput;
   enhancerGenerationConfig?: RetrievalEnhancerGenerationConfigInput;
   chunkingConfig?: RetrievalChunkingConfigInput;
+  contextPackingConfig?: RetrievalContextPackingConfigInput;
+  snippetIntegrityConfig?: RetrievalSnippetIntegrityConfigInput;
   enhancerDecisionTraceEnabled?: boolean;
 }
 
@@ -849,10 +1100,25 @@ function singularizeToken(token: string): string | undefined {
   return undefined;
 }
 
+function tokenizeLightweight(text: string): string[] {
+  return text
+    .normalize("NFKC")
+    .split(/[^A-Za-z0-9_]+/)
+    .map((token) => token.trim().toLowerCase())
+    .filter(Boolean);
+}
+
 function tokenize(text: string): string[] {
   return tokenizeForRanking(text);
 }
 
+function chunkBudgetTokenize(text: string, mode: "ranking" | "lightweight"): string[] {
+  if (mode === "lightweight") {
+    return tokenizeLightweight(text);
+  }
+  return tokenize(text);
+}
+
 function lexicalScore(query: string, haystack: string): number {
   const q = new Set(tokenize(query));
   if (q.size === 0) {
@@ -1356,6 +1622,7 @@ interface ChunkBuildReport {
   language_aware_attempt_latency_ms?: number;
   fallback_path_latency_ms?: number;
   language?: string;
+  recursive_semantic_chunking_used?: boolean;
 }
 
 function buildChunks(file: RawFile, chunkingConfig: RetrievalChunkingConfig): ChunkBuildReport {
@@ -1371,11 +1638,17 @@ function buildChunks(file: RawFile, chunkingConfig: RetrievalChunkingConfig): Ch
       fallback_strategy: chunkingConfig.fallback_strategy,
       target_chunk_tokens: chunkingConfig.target_chunk_tokens,
       chunk_overlap_tokens: chunkingConfig.chunk_overlap_tokens,
+      budget_tokenizer: chunkingConfig.budget_tokenizer,
+      boundary_strictness: chunkingConfig.boundary_strictness,
       max_chunks_per_file: MAX_CHUNKS_PER_FILE,
       parse_timeout_ms: chunkingConfig.parse_timeout_ms,
-      enabled_languages: chunkingConfig.enabled_languages
+      enabled_languages: chunkingConfig.enabled_languages,
+      recursive_semantic_chunking_enabled: chunkingConfig.recursive_semantic_chunking_enabled,
+      semantic_merge_gap_lines: chunkingConfig.semantic_merge_gap_lines,
+      semantic_merge_max_span_lines: chunkingConfig.semantic_merge_max_span_lines,
+      comment_forward_absorb_enabled: chunkingConfig.comment_forward_absorb_enabled
     },
-    tokenize
+    tokenize: (text: string) => chunkBudgetTokenize(text, chunkingConfig.budget_tokenizer)
   });
 
   return {
@@ -1394,10 +1667,38 @@ function buildChunks(file: RawFile, chunkingConfig: RetrievalChunkingConfig): Ch
     parse_latency_ms: chunkingResult.parse_latency_ms,
     language_aware_attempt_latency_ms: chunkingResult.language_aware_attempt_latency_ms,
     fallback_path_latency_ms: chunkingResult.fallback_path_latency_ms,
-    language: chunkingResult.language
+    language: chunkingResult.language,
+    recursive_semantic_chunking_used: chunkingResult.recursive_semantic_chunking_used
   };
 }
 
+function buildChunkEmbeddingText(
+  chunk: IndexedChunk,
+  config: RetrievalChunkingConfig,
+  embeddingProviderId: string
+): string {
+  const isDeterministicProvider = embeddingProviderId.trim().toLowerCase() === "deterministic";
+  if (!config.embedding_context_prefix_enabled || isDeterministicProvider) {
+    return chunk.snippet;
+  }
+  const normalizedPath = normalizePath(chunk.path);
+  const pathParts = normalizedPath.split("/").filter(Boolean);
+  const contextPath = pathParts.length > 2 ? pathParts.slice(-2).join("/") : normalizedPath;
+  const symbol = detectSnippetSymbolName(chunk.snippet);
+  const linesLabel = `${chunk.start_line}-${chunk.end_line}`;
+  const symbolLabel = symbol ? ` > ${symbol}` : "";
+  const prefix = `${contextPath}:${linesLabel}${symbolLabel}`;
+  return `${prefix}\n${chunk.snippet}`;
+}
+
+function buildChunkEmbeddingTexts(
+  chunks: IndexedChunk[],
+  config: RetrievalChunkingConfig,
+  embeddingProviderId: string
+): string[] {
+  return chunks.map((chunk) => buildChunkEmbeddingText(chunk, config, embeddingProviderId));
+}
+
 function pseudoEmbedding(input: string, dimensions = 24): number[] {
   const safeDimensions = Math.max(1, dimensions);
   let source = sha256(input);
@@ -3858,6 +4159,827 @@ function compareSearchResults(a: SearchContextOutput["results"][number], b: Sear
   return a.end_line - b.end_line;
 }
 
+function compareSearchResultsByLineRange(
+  a: SearchContextOutput["results"][number],
+  b: SearchContextOutput["results"][number]
+): number {
+  if (a.start_line !== b.start_line) {
+    return a.start_line - b.start_line;
+  }
+  if (a.end_line !== b.end_line) {
+    return a.end_line - b.end_line;
+  }
+  return compareSearchResults(a, b);
+}
+
+function mergeSnippetCluster(
+  cluster: SearchContextOutput["results"],
+  mergedStartLine: number,
+  mergedEndLine: number
+): string {
+  const byRelevance = [...cluster].sort(compareSearchResults);
+  const primary = byRelevance[0];
+  if (!primary) {
+    return "";
+  }
+  const lineMap = new Map<number, { text: string; score: number; rank: number }>();
+  for (let rank = 0; rank < byRelevance.length; rank += 1) {
+    const candidate = byRelevance[rank];
+    if (!candidate) {
+      continue;
+    }
+    const lines = candidate.snippet.replace(/\r\n/g, "\n").split("\n");
+    const expectedLineCount = Math.max(1, candidate.end_line - candidate.start_line + 1);
+    const maxLines = Math.min(lines.length, expectedLineCount);
+    for (let offset = 0; offset < maxLines; offset += 1) {
+      const lineNumber = candidate.start_line + offset;
+      if (lineNumber < mergedStartLine || lineNumber > mergedEndLine) {
+        continue;
+      }
+      const text = lines[offset];
+      if (typeof text !== "string") {
+        continue;
+      }
+      const existing = lineMap.get(lineNumber);
+      if (!existing || candidate.score > existing.score + 1e-9 || (Math.abs(candidate.score - existing.score) <= 1e-9 && rank < existing.rank)) {
+        lineMap.set(lineNumber, { text, score: candidate.score, rank });
+      }
+    }
+  }
+
+  const mergedLines: string[] = [];
+  let missingLines = 0;
+  for (let line = mergedStartLine; line <= mergedEndLine; line += 1) {
+    const entry = lineMap.get(line);
+    if (!entry) {
+      missingLines += 1;
+      mergedLines.push("");
+      continue;
+    }
+    mergedLines.push(entry.text);
+  }
+
+  const totalLines = Math.max(1, mergedEndLine - mergedStartLine + 1);
+  const maxMissingLines = Math.max(2, Math.floor(totalLines * 0.2));
+  if (missingLines > maxMissingLines) {
+    return primary.snippet;
+  }
+
+  return mergedLines.join("\n");
+}
+
+function mergeCandidateCluster(cluster: SearchContextOutput["results"]): SearchContextOutput["results"][number] {
+  if (cluster.length === 0) {
+    throw new Error("mergeCandidateCluster requires at least one candidate");
+  }
+  if (cluster.length === 1) {
+    return cluster[0]!;
+  }
+
+  const byRelevance = [...cluster].sort(compareSearchResults);
+  const primary = byRelevance[0]!;
+  const mergedStartLine = Math.min(...cluster.map((candidate) => candidate.start_line));
+  const mergedEndLine = Math.max(...cluster.map((candidate) => candidate.end_line));
+  const stitchedSnippet = mergeSnippetCluster(cluster, mergedStartLine, mergedEndLine);
+
+  return {
+    ...primary,
+    start_line: mergedStartLine,
+    end_line: mergedEndLine,
+    snippet: stitchedSnippet.length > 0 ? stitchedSnippet : primary.snippet
+  };
+}
+
+const HEAVY_LINE_RANGE_OVERLAP_RATIO = 0.2;
+
+function lineRangeLength(startLine: number, endLine: number): number {
+  return Math.max(1, endLine - startLine + 1);
+}
+
+function lineRangeOverlapLength(
+  aStartLine: number,
+  aEndLine: number,
+  bStartLine: number,
+  bEndLine: number
+): number {
+  const start = Math.max(aStartLine, bStartLine);
+  const end = Math.min(aEndLine, bEndLine);
+  if (end < start) {
+    return 0;
+  }
+  return end - start + 1;
+}
+
+function isHeavilyOverlappingLineRange(
+  candidate: SearchContextOutput["results"][number],
+  selectedRanges: Array<{ start_line: number; end_line: number }>
+): boolean {
+  for (const selected of selectedRanges) {
+    const overlapLength = lineRangeOverlapLength(
+      selected.start_line,
+      selected.end_line,
+      candidate.start_line,
+      candidate.end_line
+    );
+    if (overlapLength <= 0) {
+      continue;
+    }
+    const smallerRange = Math.min(
+      lineRangeLength(selected.start_line, selected.end_line),
+      lineRangeLength(candidate.start_line, candidate.end_line)
+    );
+    const overlapRatio = overlapLength / Math.max(1, smallerRange);
+    if (overlapRatio >= HEAVY_LINE_RANGE_OVERLAP_RATIO) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function mergeLineSpans(spans: Array<{ start_line: number; end_line: number }>): Array<{ start_line: number; end_line: number }> {
+  if (spans.length <= 1) {
+    return [...spans];
+  }
+  const ordered = [...spans]
+    .filter((span) => span.end_line >= span.start_line)
+    .sort((a, b) => a.start_line - b.start_line || a.end_line - b.end_line);
+  const merged: Array<{ start_line: number; end_line: number }> = [];
+  for (const span of ordered) {
+    const last = merged[merged.length - 1];
+    if (!last || span.start_line > last.end_line + 1) {
+      merged.push({ ...span });
+      continue;
+    }
+    last.end_line = Math.max(last.end_line, span.end_line);
+  }
+  return merged;
+}
+
+function lineRangeGap(anchor: { start_line: number; end_line: number }, candidate: { start_line: number; end_line: number }): number {
+  if (candidate.start_line > anchor.end_line) {
+    return candidate.start_line - anchor.end_line - 1;
+  }
+  if (anchor.start_line > candidate.end_line) {
+    return anchor.start_line - candidate.end_line - 1;
+  }
+  return 0;
+}
+
+function buildPreferredLineMap(candidates: SearchContextOutput["results"]): Map<number, string> {
+  const byRelevance = [...candidates].sort(compareSearchResults);
+  const lineMap = new Map<number, { text: string; score: number; rank: number }>();
+  for (let rank = 0; rank < byRelevance.length; rank += 1) {
+    const candidate = byRelevance[rank];
+    if (!candidate) {
+      continue;
+    }
+    const lines = candidate.snippet.replace(/\r\n/g, "\n").split("\n");
+    const expectedLineCount = Math.max(1, candidate.end_line - candidate.start_line + 1);
+    const maxLines = Math.min(lines.length, expectedLineCount);
+    for (let offset = 0; offset < maxLines; offset += 1) {
+      const lineNumber = candidate.start_line + offset;
+      const text = lines[offset];
+      if (typeof text !== "string") {
+        continue;
+      }
+      const existing = lineMap.get(lineNumber);
+      if (!existing || candidate.score > existing.score + 1e-9 || (Math.abs(candidate.score - existing.score) <= 1e-9 && rank < existing.rank)) {
+        lineMap.set(lineNumber, { text, score: candidate.score, rank });
+      }
+    }
+  }
+  return new Map([...lineMap.entries()].map(([line, value]) => [line, value.text]));
+}
+
+function clipSnippetToMaxChars(snippet: string, maxChars: number): string {
+  if (snippet.length <= maxChars) {
+    return snippet;
+  }
+  const clipped = snippet.slice(0, Math.max(0, maxChars));
+  const lastNewline = clipped.lastIndexOf("\n");
+  if (lastNewline > 80) {
+    return clipped.slice(0, lastNewline).trimEnd();
+  }
+  return clipped.trimEnd();
+}
+
+function snippetIntegrityLanguageFromPath(path: string): string | undefined {
+  const extension = fileExtension(path);
+  if (extension === ".ts" || extension === ".mts" || extension === ".cts") {
+    return "typescript";
+  }
+  if (extension === ".tsx") {
+    return "tsx";
+  }
+  if (extension === ".js" || extension === ".mjs" || extension === ".cjs") {
+    return "javascript";
+  }
+  if (extension === ".jsx") {
+    return "jsx";
+  }
+  return undefined;
+}
+
+function firstNonEmptyLine(snippet: string): string {
+  const lines = snippet.replace(/\r\n/g, "\n").split("\n");
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.length > 0) {
+      return trimmed;
+    }
+  }
+  return "";
+}
+
+function lastNonEmptyLine(snippet: string): string {
+  const lines = snippet.replace(/\r\n/g, "\n").split("\n");
+  for (let idx = lines.length - 1; idx >= 0; idx -= 1) {
+    const trimmed = (lines[idx] ?? "").trim();
+    if (trimmed.length > 0) {
+      return trimmed;
+    }
+  }
+  return "";
+}
+
+function curlyBraceDelta(snippet: string): number {
+  let opens = 0;
+  let closes = 0;
+  for (const char of snippet) {
+    if (char === "{") {
+      opens += 1;
+      continue;
+    }
+    if (char === "}") {
+      closes += 1;
+    }
+  }
+  return opens - closes;
+}
+
+function looksLikeDeclarationStart(line: string): boolean {
+  if (line.length === 0) {
+    return false;
+  }
+  if (line.startsWith("@")) {
+    return true;
+  }
+  return (
+    /^(?:export\s+)?(?:async\s+)?function\s+[A-Za-z_$][\w$]*\s*\(/u.test(line) ||
+    /^(?:export\s+)?(?:default\s+)?class\s+[A-Za-z_$][\w$]*/u.test(line) ||
+    /^(?:export\s+)?(?:const|let|var)\s+[A-Za-z_$][\w$]*\s*=/u.test(line) ||
+    /^(?:public|private|protected|static|readonly|async)\s+[A-Za-z_$][\w$]*\s*\(/u.test(line) ||
+    /^(?:[A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{/u.test(line)
+  );
+}
+
+function looksLikeSnippetTerminalBoundary(line: string): boolean {
+  if (line.length === 0) {
+    return false;
+  }
+  return (
+    line.endsWith("}") ||
+    line.endsWith("};") ||
+    line.endsWith(");") ||
+    line.endsWith("]") ||
+    line.endsWith("];")
+  );
+}
+
+function detectSnippetSymbolName(snippet: string): string | undefined {
+  const lines = snippet.replace(/\r\n/g, "\n").split("\n").slice(0, 40);
+  const patterns = [
+    /^(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\(/u,
+    /^(?:export\s+)?(?:default\s+)?class\s+([A-Za-z_$][\w$]*)\b/u,
+    /^(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\([^)]*\)\s*=>/u,
+    /^(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?[A-Za-z_$][\w$]*\s*=>/u,
+    /^(?:public|private|protected|static|readonly|async)\s+([A-Za-z_$][\w$]*)\s*\(/u,
+    /^([A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{/u
+  ];
+  const disallowed = new Set(["if", "for", "while", "switch", "catch", "return"]);
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) {
+      continue;
+    }
+    for (const pattern of patterns) {
+      const match = trimmed.match(pattern);
+      const symbol = match?.[1];
+      if (symbol && !disallowed.has(symbol)) {
+        return symbol;
+      }
+    }
+  }
+  return undefined;
+}
+
+function shouldAnnotateSnippetAsTruncated(
+  result: SearchResultRow,
+  omittedBefore: number,
+  omittedAfter: number
+): boolean {
+  if (omittedBefore <= 0 && omittedAfter <= 0) {
+    return false;
+  }
+  const firstLine = firstNonEmptyLine(result.snippet);
+  const lastLine = lastNonEmptyLine(result.snippet);
+  if (omittedBefore > 0 && !looksLikeDeclarationStart(firstLine)) {
+    return true;
+  }
+  if (omittedAfter > 0) {
+    if (curlyBraceDelta(result.snippet) > 0) {
+      return true;
+    }
+    if (!looksLikeSnippetTerminalBoundary(lastLine)) {
+      return true;
+    }
+  }
+  return omittedBefore > 0 && omittedAfter > 0;
+}
+
+function estimateContiguousEnvelope(input: {
+  anchor: SearchResultRow;
+  candidates: SearchResultRow[];
+  maxGapLines: number;
+}): { start_line: number; end_line: number } {
+  let start = input.anchor.start_line;
+  let end = input.anchor.end_line;
+  let changed = true;
+  while (changed) {
+    changed = false;
+    for (const candidate of input.candidates) {
+      const gap = lineRangeGap({ start_line: start, end_line: end }, candidate);
+      if (gap > input.maxGapLines) {
+        continue;
+      }
+      const nextStart = Math.min(start, candidate.start_line);
+      const nextEnd = Math.max(end, candidate.end_line);
+      if (nextStart !== start || nextEnd !== end) {
+        start = nextStart;
+        end = nextEnd;
+        changed = true;
+      }
+    }
+  }
+  return { start_line: start, end_line: end };
+}
+
+function repairSnippetFromEnvelope(input: {
+  anchor: SearchResultRow;
+  envelope: { start_line: number; end_line: number };
+  samePathCandidates: SearchResultRow[];
+  config: RetrievalSnippetIntegrityConfig;
+}): { repaired?: SearchResultRow; reason?: string; clipped: boolean } {
+  const envelopeSpan = input.envelope.end_line - input.envelope.start_line + 1;
+  if (envelopeSpan > input.config.repair_max_envelope_lines) {
+    return { reason: "envelope_cap_exceeded", clipped: false };
+  }
+
+  const envelopeCandidates = input.samePathCandidates
+    .filter((candidate) => candidate.end_line >= input.envelope.start_line && candidate.start_line <= input.envelope.end_line)
+    .sort(compareSearchResultsByLineRange);
+  if (envelopeCandidates.length === 0) {
+    return { reason: "no_envelope_candidates", clipped: false };
+  }
+
+  const lineMap = buildPreferredLineMap(envelopeCandidates);
+  const renderedLines: string[] = [];
+  let missingLines = 0;
+  for (let line = input.envelope.start_line; line <= input.envelope.end_line; line += 1) {
+    const text = lineMap.get(line);
+    if (typeof text !== "string") {
+      missingLines += 1;
+      renderedLines.push("");
+      continue;
+    }
+    renderedLines.push(text);
+  }
+
+  const maxMissingLines = Math.max(2, Math.floor(envelopeSpan * 0.2));
+  if (missingLines > maxMissingLines) {
+    return { reason: "missing_line_density_too_high", clipped: false };
+  }
+
+  const clippedLines: string[] = [];
+  let usedChars = 0;
+  let clipped = false;
+  for (let index = 0; index < renderedLines.length; index += 1) {
+    const line = renderedLines[index] ?? "";
+    const additionalChars = index === 0 ? line.length : line.length + 1;
+    if (clippedLines.length > 0 && usedChars + additionalChars > input.config.repair_max_snippet_chars) {
+      clipped = true;
+      break;
+    }
+    if (clippedLines.length === 0 && line.length > input.config.repair_max_snippet_chars) {
+      const clippedLine = line.slice(0, input.config.repair_max_snippet_chars);
+      if (clippedLine.length === 0) {
+        return { reason: "snippet_char_cap_exceeded", clipped: false };
+      }
+      clippedLines.push(clippedLine);
+      usedChars = clippedLine.length;
+      clipped = true;
+      break;
+    }
+    clippedLines.push(line);
+    usedChars += additionalChars;
+  }
+
+  if (clippedLines.length === 0) {
+    return { reason: "snippet_char_cap_exceeded", clipped: false };
+  }
+
+  const repairedSnippet = clippedLines.join("\n").trimEnd();
+  if (repairedSnippet.length === 0) {
+    return { reason: "empty_repaired_snippet", clipped: false };
+  }
+
+  const repairedEndLine = input.envelope.start_line + clippedLines.length - 1;
+  return {
+    repaired: {
+      ...input.anchor,
+      start_line: input.envelope.start_line,
+      end_line: repairedEndLine,
+      snippet: repairedSnippet
+    },
+    clipped
+  };
+}
+
+function buildSnippetTruncationMarker(input: {
+  result: SearchResultRow;
+  symbolName?: string;
+  envelope_start_line: number;
+  envelope_end_line: number;
+  marker_template_version: "v1";
+}): string {
+  const estimatedTotalLines = Math.max(1, input.envelope_end_line - input.envelope_start_line + 1);
+  const omittedBefore = Math.max(0, input.result.start_line - input.envelope_start_line);
+  const omittedAfter = Math.max(0, input.envelope_end_line - input.result.end_line);
+  return `// [truncated:${input.marker_template_version} symbol=${input.symbolName ?? "unknown"} estimated_span=${
+    input.envelope_start_line
+  }-${input.envelope_end_line} estimated_total_lines=${estimatedTotalLines} omitted_before=${omittedBefore} omitted_after=${omittedAfter} through_line=${input.result.end_line}]`;
+}
+
+function annotateSearchResultsWithSnippetIntegrity(input: {
+  selected: SearchResultRow[];
+  sourceCandidates: SearchResultRow[];
+  config: RetrievalSnippetIntegrityConfig;
+  observability: Observability;
+  retrievalProfileId: string;
+}): SearchResultRow[] {
+  if (!input.config.enabled || input.selected.length === 0) {
+    return [...input.selected];
+  }
+
+  const enabledLanguages = new Set(normalizeSnippetIntegrityLanguageList(input.config.target_languages));
+  if (enabledLanguages.size === 0) {
+    return [...input.selected];
+  }
+
+  const sourceByPath = new Map<string, SearchResultRow[]>();
+  for (const candidate of input.sourceCandidates) {
+    const rows = sourceByPath.get(candidate.path);
+    if (rows) {
+      rows.push(candidate);
+    } else {
+      sourceByPath.set(candidate.path, [candidate]);
+    }
+  }
+
+  return input.selected.map((result) => {
+    const language = snippetIntegrityLanguageFromPath(result.path);
+    if (!language || !enabledLanguages.has(language)) {
+      return result;
+    }
+    const samePath = sourceByPath.get(result.path) ?? [result];
+    if (samePath.length <= 1) {
+      return result;
+    }
+    const envelope = estimateContiguousEnvelope({
+      anchor: result,
+      candidates: samePath,
+      maxGapLines: input.config.max_contiguous_gap_lines
+    });
+    const originalOmittedBefore = Math.max(0, result.start_line - envelope.start_line);
+    const originalOmittedAfter = Math.max(0, envelope.end_line - result.end_line);
+    const originalLooksTruncated = shouldAnnotateSnippetAsTruncated(result, originalOmittedBefore, originalOmittedAfter);
+    if (!originalLooksTruncated) {
+      return result;
+    }
+
+    const envelopeCandidates = samePath
+      .filter((candidate) => candidate.end_line >= envelope.start_line && candidate.start_line <= envelope.end_line)
+      .sort(compareSearchResultsByLineRange);
+    let assembled = result;
+
+    if (input.config.repair_enabled) {
+      input.observability.metrics.increment("retrieval_snippet_repair_attempt_total", 1, {
+        retrieval_profile_id: input.retrievalProfileId,
+        language
+      });
+      const repairOutcome = repairSnippetFromEnvelope({
+        anchor: result,
+        envelope,
+        samePathCandidates: samePath,
+        config: input.config
+      });
+      if (repairOutcome.repaired) {
+        assembled = repairOutcome.repaired;
+        input.observability.metrics.increment("retrieval_snippet_repair_success_total", 1, {
+          retrieval_profile_id: input.retrievalProfileId,
+          language,
+          clipped: repairOutcome.clipped ? "true" : "false"
+        });
+        input.observability.logger.info("snippet integrity repair decision", {
+          retrieval_profile_id: input.retrievalProfileId,
+          path: result.path,
+          language,
+          envelope_start_line: envelope.start_line,
+          envelope_end_line: envelope.end_line,
+          envelope_span_lines: envelope.end_line - envelope.start_line + 1,
+          status: "repaired",
+          clipped: repairOutcome.clipped
+        });
+      } else {
+        input.observability.logger.info("snippet integrity repair decision", {
+          retrieval_profile_id: input.retrievalProfileId,
+          path: result.path,
+          language,
+          envelope_start_line: envelope.start_line,
+          envelope_end_line: envelope.end_line,
+          envelope_span_lines: envelope.end_line - envelope.start_line + 1,
+          status: "repair_skipped",
+          reason: repairOutcome.reason ?? "unknown"
+        });
+      }
+    }
+
+    const omittedBefore = Math.max(0, assembled.start_line - envelope.start_line);
+    const omittedAfter = Math.max(0, envelope.end_line - assembled.end_line);
+    if (!shouldAnnotateSnippetAsTruncated(assembled, omittedBefore, omittedAfter)) {
+      return assembled;
+    }
+
+    let symbolName = detectSnippetSymbolName(assembled.snippet);
+    if (!symbolName) {
+      for (const candidate of envelopeCandidates) {
+        symbolName = detectSnippetSymbolName(candidate.snippet);
+        if (symbolName) {
+          break;
+        }
+      }
+    }
+    const marker = buildSnippetTruncationMarker({
+      result: assembled,
+      symbolName,
+      envelope_start_line: envelope.start_line,
+      envelope_end_line: envelope.end_line,
+      marker_template_version: input.config.marker_template_version
+    });
+    input.observability.metrics.increment("retrieval_snippet_repair_fallback_marker_total", 1, {
+      retrieval_profile_id: input.retrievalProfileId,
+      language
+    });
+    input.observability.metrics.increment("retrieval_snippet_truncation_marker_total", 1, {
+      retrieval_profile_id: input.retrievalProfileId,
+      language,
+      symbol_detected: symbolName ? "true" : "false",
+      marker_template_version: input.config.marker_template_version
+    });
+    input.observability.metrics.observe("retrieval_snippet_omitted_after_lines", omittedAfter, {
+      retrieval_profile_id: input.retrievalProfileId,
+      language
+    });
+    const baseSnippet = assembled.snippet.trimEnd();
+    return {
+      ...assembled,
+      snippet: baseSnippet.length > 0 ? `${baseSnippet}\n${marker}` : marker
+    };
+  });
+}
+
+function packSearchResultsWithContext(input: {
+  selected: SearchContextOutput["results"];
+  sourceCandidates: SearchContextOutput["results"];
+  config: RetrievalContextPackingConfig;
+}): SearchContextOutput["results"] {
+  if (!input.config.enabled || input.selected.length === 0) {
+    return [...input.selected];
+  }
+
+  const sourceByPath = new Map<string, SearchContextOutput["results"]>();
+  for (const candidate of input.sourceCandidates) {
+    const rows = sourceByPath.get(candidate.path);
+    if (rows) {
+      rows.push(candidate);
+    } else {
+      sourceByPath.set(candidate.path, [candidate]);
+    }
+  }
+
+  return input.selected.map((anchor) => {
+    const samePath = sourceByPath.get(anchor.path) ?? [anchor];
+    if (samePath.length <= 1 || input.config.max_spans_per_result <= 1) {
+      return anchor;
+    }
+
+    const anchorRange = { start_line: anchor.start_line, end_line: anchor.end_line };
+    const candidates = samePath
+      .filter(
+        (candidate) =>
+          !(candidate.start_line === anchor.start_line && candidate.end_line === anchor.end_line) &&
+          !isHeavilyOverlappingLineRange(candidate, [anchorRange]) &&
+          lineRangeGap(anchorRange, candidate) <= input.config.max_gap_lines
+      )
+      .sort((a, b) => {
+        const relevanceDiff = compareSearchResults(a, b);
+        if (relevanceDiff !== 0) {
+          return relevanceDiff;
+        }
+        return lineRangeGap(anchorRange, a) - lineRangeGap(anchorRange, b);
+      });
+
+    const spans: Array<{ start_line: number; end_line: number }> = [{ ...anchorRange }];
+    for (const candidate of candidates) {
+      if (spans.length >= input.config.max_spans_per_result) {
+        break;
+      }
+      const nextSpan = { start_line: candidate.start_line, end_line: candidate.end_line };
+      const nextEnvelope = mergeLineSpans([...spans, nextSpan]);
+      if (
+        nextEnvelope.some((span, idx) => idx > 0 && span.start_line - (nextEnvelope[idx - 1]?.end_line ?? span.start_line) - 1 > input.config.max_gap_lines)
+      ) {
+        continue;
+      }
+      spans.push(nextSpan);
+    }
+
+    const mergedSpans = mergeLineSpans(spans);
+    if (mergedSpans.length <= 1) {
+      return anchor;
+    }
+
+    const lineMap = buildPreferredLineMap([anchor, ...samePath]);
+    const renderedLines: string[] = [];
+    let contentLineCount = 0;
+    let elisionCount = 0;
+    for (let index = 0; index < mergedSpans.length; index += 1) {
+      const span = mergedSpans[index];
+      if (!span) {
+        continue;
+      }
+      if (index > 0) {
+        const previous = mergedSpans[index - 1];
+        if (previous && span.start_line - previous.end_line > 0) {
+          renderedLines.push("...");
+          elisionCount += 1;
+        }
+      }
+      for (let line = span.start_line; line <= span.end_line; line += 1) {
+        renderedLines.push(lineMap.get(line) ?? "");
+        contentLineCount += 1;
+      }
+    }
+    if (renderedLines.length === 0) {
+      return anchor;
+    }
+
+    const elisionDensity = elisionCount / Math.max(1, contentLineCount + elisionCount);
+    if (elisionDensity > 0.25) {
+      return anchor;
+    }
+
+    const packedSnippet = clipSnippetToMaxChars(renderedLines.join("\n"), input.config.max_snippet_chars);
+    if (packedSnippet.length === 0) {
+      return anchor;
+    }
+    const packedStart = mergedSpans[0]?.start_line ?? anchor.start_line;
+    const packedEnd = mergedSpans[mergedSpans.length - 1]?.end_line ?? anchor.end_line;
+
+    return {
+      ...anchor,
+      start_line: packedStart,
+      end_line: packedEnd,
+      snippet: packedSnippet,
+      reason: `${anchor.reason} + contextual spans`
+    };
+  });
+}
+
+function mergeOverlappingCandidates(
+  candidates: SearchContextOutput["results"],
+  config: RetrievalRerankConfig
+): SearchContextOutput["results"] {
+  if (!config.merge_overlapping_chunks_enabled || candidates.length <= 1) {
+    return [...candidates];
+  }
+
+  const byPath = new Map<string, SearchContextOutput["results"]>();
+  for (const candidate of candidates) {
+    const group = byPath.get(candidate.path);
+    if (group) {
+      group.push(candidate);
+    } else {
+      byPath.set(candidate.path, [candidate]);
+    }
+  }
+
+  const merged: SearchContextOutput["results"] = [];
+  for (const group of byPath.values()) {
+    const ordered = [...group].sort(compareSearchResultsByLineRange);
+    let cluster: SearchContextOutput["results"] = [];
+    let clusterStart = 0;
+    let clusterEnd = 0;
+
+    const flush = (): void => {
+      if (cluster.length === 0) {
+        return;
+      }
+      merged.push(mergeCandidateCluster(cluster));
+      cluster = [];
+    };
+
+    for (const candidate of ordered) {
+      if (cluster.length === 0) {
+        cluster = [candidate];
+        clusterStart = candidate.start_line;
+        clusterEnd = candidate.end_line;
+        continue;
+      }
+
+      const nextStart = Math.min(clusterStart, candidate.start_line);
+      const nextEnd = Math.max(clusterEnd, candidate.end_line);
+      const nextSpan = nextEnd - nextStart + 1;
+      const gapLines = Math.max(0, candidate.start_line - clusterEnd - 1);
+      const canMerge = gapLines <= config.merge_gap_lines && nextSpan <= config.merge_max_span_lines;
+
+      if (!canMerge) {
+        flush();
+        cluster = [candidate];
+        clusterStart = candidate.start_line;
+        clusterEnd = candidate.end_line;
+        continue;
+      }
+
+      cluster.push(candidate);
+      clusterStart = nextStart;
+      clusterEnd = nextEnd;
+    }
+
+    flush();
+  }
+
+  return merged.sort(compareSearchResults);
+}
+
+function applySmartCutoffCandidates(
+  candidates: SearchContextOutput["results"],
+  config: RetrievalRerankConfig
+): SearchContextOutput["results"] {
+  if (!config.smart_cutoff_enabled || candidates.length === 0) {
+    return [...candidates];
+  }
+  const ordered = [...candidates].sort(compareSearchResults);
+  const minK = Math.max(1, config.smart_cutoff_min_k);
+  const maxK = Math.max(minK, config.smart_cutoff_max_k);
+  const topScore = ordered[0]?.score ?? Number.NEGATIVE_INFINITY;
+  const kept: SearchContextOutput["results"] = [];
+
+  for (let index = 0; index < ordered.length; index += 1) {
+    const candidate = ordered[index];
+    if (!candidate) {
+      continue;
+    }
+    if (kept.length >= maxK) {
+      break;
+    }
+    if (kept.length < minK) {
+      kept.push(candidate);
+      continue;
+    }
+    if (candidate.score < config.smart_cutoff_min_score) {
+      break;
+    }
+    if (candidate.score < topScore * config.smart_cutoff_top_ratio) {
+      break;
+    }
+    const previous = ordered[index - 1];
+    if (previous && previous.score - candidate.score > config.smart_cutoff_delta_abs) {
+      break;
+    }
+    kept.push(candidate);
+  }
+
+  return kept;
+}
+
+export function __applySmartCutoffCandidatesForTests(input: {
+  candidates: SearchContextOutput["results"];
+  config: RetrievalRerankConfig;
+}): SearchContextOutput["results"] {
+  return applySmartCutoffCandidates(input.candidates, input.config);
+}
+
 function dedupeEnhancerCandidatesByPath(results: SearchContextOutput["results"]): SearchContextOutput["results"] {
   const byPath = new Map<string, SearchContextOutput["results"][number]>();
   for (const result of results) {
@@ -4248,11 +5370,14 @@ function deterministicEnhancerFallbackRanking(input: {
   return [...preferred, ...tolerated, ...avoided];
 }
 
-function trimToContextBudget(results: SearchContextOutput["results"]): SearchContextOutput["results"] {
+function trimToContextBudget(
+  results: SearchContextOutput["results"],
+  budgetTokenizerMode: "ranking" | "lightweight"
+): SearchContextOutput["results"] {
   let total = 0;
   const out: SearchContextOutput["results"] = [];
   for (const result of results) {
-    total += tokenize(result.snippet).length;
+    total += chunkBudgetTokenize(result.snippet, budgetTokenizerMode).length;
     if (total > MAX_CONTEXT_BUDGET_TOKENS) {
       break;
     }
@@ -4857,6 +5982,8 @@ export class RetrievalCore {
   private readonly enhancerConfig: RetrievalEnhancerConfig;
   private readonly enhancerGenerationConfig: RetrievalEnhancerGenerationConfig;
   private readonly chunkingConfig: RetrievalChunkingConfig;
+  private readonly contextPackingConfig: RetrievalContextPackingConfig;
+  private readonly snippetIntegrityConfig: RetrievalSnippetIntegrityConfig;
   private readonly enhancerDecisionTraceEnabled: boolean;
   private cacheHits = 0;
   private cacheMisses = 0;
@@ -4898,6 +6025,14 @@ export class RetrievalCore {
       options?.enhancerGenerationConfig
     );
     this.chunkingConfig = mergeRetrievalChunkingConfig(DEFAULT_RETRIEVAL_CHUNKING_CONFIG, options?.chunkingConfig);
+    this.contextPackingConfig = mergeRetrievalContextPackingConfig(
+      DEFAULT_RETRIEVAL_CONTEXT_PACKING_CONFIG,
+      options?.contextPackingConfig
+    );
+    this.snippetIntegrityConfig = mergeRetrievalSnippetIntegrityConfig(
+      DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG,
+      options?.snippetIntegrityConfig
+    );
     this.enhancerDecisionTraceEnabled = Boolean(options?.enhancerDecisionTraceEnabled);
   }
 
@@ -5092,6 +6227,12 @@ export class RetrievalCore {
           language: chunkLanguage,
           reason: chunkBuild.fallback_reason ?? "none"
         });
+        if (chunkBuild.recursive_semantic_chunking_used) {
+          this.observability.metrics.increment("index_recursive_semantic_chunking_used_total", 1, {
+            tenant_id: artifact.tenant_id,
+            language: chunkLanguage
+          });
+        }
         if (chunkBuild.fallback_reason) {
           this.observability.metrics.increment("index_chunking_fallback_total", 1, {
             tenant_id: artifact.tenant_id,
@@ -5123,7 +6264,8 @@ export class RetrievalCore {
             reason: chunkBuild.fallback_reason
           });
         }
-        const estimatedEmbeddingTokens = chunks.reduce((sum, chunk) => sum + tokenize(chunk.snippet).length, 0);
+        const embeddingTexts = buildChunkEmbeddingTexts(chunks, this.chunkingConfig, this.embeddingDescriptor.provider);
+        const estimatedEmbeddingTokens = embeddingTexts.reduce((sum, text) => sum + tokenize(text).length, 0);
         this.observability.metrics.increment("index_embedding_tokens_total", estimatedEmbeddingTokens, {
           tenant_id: artifact.tenant_id
         });
@@ -5131,7 +6273,7 @@ export class RetrievalCore {
           chunks.length === 0
             ? []
             : await this.embeddingProvider.embed({
-                texts: chunks.map((chunk) => chunk.snippet),
+                texts: embeddingTexts,
                 purpose: "index"
               });
         if (embeddings.length !== chunks.length) {
@@ -5449,6 +6591,12 @@ export class RetrievalCore {
           language: chunkLanguage,
           reason: chunkBuild.fallback_reason ?? "none"
         });
+        if (chunkBuild.recursive_semantic_chunking_used) {
+          this.observability.metrics.increment("index_recursive_semantic_chunking_used_total", 1, {
+            tenant_id: artifact.tenant_id,
+            language: chunkLanguage
+          });
+        }
         if (chunkBuild.fallback_reason) {
           this.observability.metrics.increment("index_chunking_fallback_total", 1, {
             tenant_id: artifact.tenant_id,
@@ -5480,7 +6628,8 @@ export class RetrievalCore {
             reason: chunkBuild.fallback_reason
           });
         }
-        const estimatedEmbeddingTokens = chunks.reduce((sum, chunk) => sum + tokenize(chunk.snippet).length, 0);
+        const embeddingTexts = buildChunkEmbeddingTexts(chunks, this.chunkingConfig, this.embeddingDescriptor.provider);
+        const estimatedEmbeddingTokens = embeddingTexts.reduce((sum, text) => sum + tokenize(text).length, 0);
         this.observability.metrics.increment("index_embedding_tokens_total", estimatedEmbeddingTokens, {
           tenant_id: artifact.tenant_id
         });
@@ -5488,7 +6637,7 @@ export class RetrievalCore {
           chunks.length === 0
             ? []
             : await this.embeddingProvider.embed({
-                texts: chunks.map((chunk) => chunk.snippet),
+                texts: embeddingTexts,
                 purpose: "index"
               });
         if (embeddings.length !== chunks.length) {
@@ -5823,7 +6972,27 @@ export class RetrievalCore {
       query,
       top_k: topK,
       filters: input.request.filters,
-      retrieval_variant: this.rerankerCacheVariant
+      retrieval_variant: `${this.rerankerCacheVariant}|context_pack:${
+        this.contextPackingConfig.enabled ? "on" : "off"
+      }|context_pack_spans:${this.contextPackingConfig.max_spans_per_result}|context_pack_gap:${this.contextPackingConfig.max_gap_lines}|snippet_integrity:${
+        this.snippetIntegrityConfig.enabled ? "on" : "off"
+      }|snippet_integrity_gap:${this.snippetIntegrityConfig.max_contiguous_gap_lines}|snippet_integrity_langs:${this.snippetIntegrityConfig.target_languages.join(
+        ","
+      )}|snippet_repair:${this.snippetIntegrityConfig.repair_enabled ? "on" : "off"}|snippet_repair_env:${
+        this.snippetIntegrityConfig.repair_max_envelope_lines
+      }|snippet_repair_chars:${this.snippetIntegrityConfig.repair_max_snippet_chars}|chunk_recursive:${
+        this.chunkingConfig.recursive_semantic_chunking_enabled ? "on" : "off"
+      }|chunk_semantic_gap:${this.chunkingConfig.semantic_merge_gap_lines}|chunk_semantic_span:${
+        this.chunkingConfig.semantic_merge_max_span_lines
+      }|chunk_comment_absorb:${this.chunkingConfig.comment_forward_absorb_enabled ? "on" : "off"}|chunk_embed_prefix:${
+        this.chunkingConfig.embedding_context_prefix_enabled ? "on" : "off"
+      }|smart_cutoff:${this.scoringConfig.rerank.smart_cutoff_enabled ? "on" : "off"}|smart_cutoff_min_k:${
+        this.scoringConfig.rerank.smart_cutoff_min_k
+      }|smart_cutoff_max_k:${this.scoringConfig.rerank.smart_cutoff_max_k}|smart_cutoff_min_score:${
+        this.scoringConfig.rerank.smart_cutoff_min_score
+      }|smart_cutoff_top_ratio:${this.scoringConfig.rerank.smart_cutoff_top_ratio}|smart_cutoff_delta_abs:${
+        this.scoringConfig.rerank.smart_cutoff_delta_abs
+      }`
     });
 
     const cached = await this.cache.get(cacheKey);
@@ -5988,17 +7157,47 @@ export class RetrievalCore {
         })
     );
 
+    const consolidatedCandidates = await this.observability.tracing.withSpan(
+      "retrieval.overlap_merge",
+      { trace_id: input.trace_id },
+      async () => mergeOverlappingCandidates(rerankedCandidates, this.scoringConfig.rerank)
+    );
+    this.observability.metrics.observe("retrieval_candidates_post_overlap_merge_count", consolidatedCandidates.length, {
+      retrieval_profile_id: this.scoringProfileId
+    });
+    const mergedCandidateCount = Math.max(0, rerankedCandidates.length - consolidatedCandidates.length);
+    if (mergedCandidateCount > 0) {
+      this.observability.metrics.increment("retrieval_overlap_candidates_merged_total", mergedCandidateCount, {
+        retrieval_profile_id: this.scoringProfileId
+      });
+    }
+    const cutoffCandidates = await this.observability.tracing.withSpan(
+      "retrieval.smart_cutoff",
+      { trace_id: input.trace_id },
+      async () => applySmartCutoffCandidates(consolidatedCandidates, this.scoringConfig.rerank)
+    );
+    if (this.scoringConfig.rerank.smart_cutoff_enabled) {
+      this.observability.metrics.increment("retrieval_smart_cutoff_applied_total", 1, {
+        retrieval_profile_id: this.scoringProfileId
+      });
+      const droppedCount = Math.max(0, consolidatedCandidates.length - cutoffCandidates.length);
+      this.observability.metrics.increment("retrieval_smart_cutoff_drop_count", droppedCount, {
+        retrieval_profile_id: this.scoringProfileId
+      });
+    }
+
     const deduped = await this.observability.tracing.withSpan("retrieval.rerank", { trace_id: input.trace_id }, async () => {
       const output: SearchContextOutput["results"] = [];
       const seen = new Set<string>();
       const pathCounts = new Map<string, number>();
+      const selectedRangesByPath = new Map<string, Array<{ start_line: number; end_line: number }>>();
       const directoryCounts = new Map<string, number>();
       const extensionCounts = new Map<string, number>();
       const maxChunksPerPath = hasFileLookupIntent(queryTokens)
         ? this.scoringConfig.rerank.max_chunks_per_path_file_lookup
         : this.scoringConfig.rerank.max_chunks_per_path_default;
 
-      const available = [...rerankedCandidates];
+      const available = [...cutoffCandidates];
       while (output.length < topK && available.length > 0) {
         let bestIndex = -1;
         let bestAdjustedScore = Number.NEGATIVE_INFINITY;
@@ -6018,6 +7217,12 @@ export class RetrievalCore {
           if (pathCount >= maxChunksPerPath) {
             continue;
           }
+          if (this.scoringConfig.rerank.merge_overlapping_chunks_enabled && pathCount > 0) {
+            const selectedRanges = selectedRangesByPath.get(candidate.path) ?? [];
+            if (isHeavilyOverlappingLineRange(candidate, selectedRanges)) {
+              continue;
+            }
+          }
 
           const directoryKey = parentDirectory(candidate.path).toLowerCase();
           const extensionKey = fileExtension(candidate.path);
@@ -6057,6 +7262,12 @@ export class RetrievalCore {
         const selectedKey = `${selected.path}:${selected.start_line}:${selected.end_line}`;
         seen.add(selectedKey);
         pathCounts.set(selected.path, (pathCounts.get(selected.path) ?? 0) + 1);
+        const selectedRanges = selectedRangesByPath.get(selected.path);
+        if (selectedRanges) {
+          selectedRanges.push({ start_line: selected.start_line, end_line: selected.end_line });
+        } else {
+          selectedRangesByPath.set(selected.path, [{ start_line: selected.start_line, end_line: selected.end_line }]);
+        }
         const selectedDirectory = parentDirectory(selected.path).toLowerCase();
         const selectedExtension = fileExtension(selected.path);
         directoryCounts.set(selectedDirectory, (directoryCounts.get(selectedDirectory) ?? 0) + 1);
@@ -6067,8 +7278,8 @@ export class RetrievalCore {
     });
 
     const candidateRankByKey = new Map<string, number>();
-    for (let index = 0; index < rerankedCandidates.length; index += 1) {
-      const candidate = rerankedCandidates[index];
+    for (let index = 0; index < cutoffCandidates.length; index += 1) {
+      const candidate = cutoffCandidates[index];
       if (!candidate) {
         continue;
       }
@@ -6100,10 +7311,22 @@ export class RetrievalCore {
     this.observability.metrics.observe("retrieval_literal_matches_topk", literalMatchesInTopK, {
       retrieval_profile_id: this.scoringProfileId
     });
+    const packedResults = packSearchResultsWithContext({
+      selected: deduped,
+      sourceCandidates: cutoffCandidates,
+      config: this.contextPackingConfig
+    });
+    const assembledResults = annotateSearchResultsWithSnippetIntegrity({
+      selected: packedResults,
+      sourceCandidates: cutoffCandidates,
+      config: this.snippetIntegrityConfig,
+      observability: this.observability,
+      retrievalProfileId: this.scoringProfileId
+    });
 
     const output: SearchContextOutput = {
       trace_id: input.trace_id,
-      results: deduped,
+      results: assembledResults,
       search_metadata: {
         latency_ms: Date.now() - searchStartedAt,
         retrieval_mode: "hybrid" satisfies RetrievalMode,
@@ -6111,7 +7334,7 @@ export class RetrievalCore {
       }
     };
 
-    this.observability.metrics.observe("retrieval_topk_hit_proxy", deduped.length > 0 ? 1 : 0, {
+    this.observability.metrics.observe("retrieval_topk_hit_proxy", assembledResults.length > 0 ? 1 : 0, {
       retrieval_profile_id: this.scoringProfileId
     });
     this.observability.logger.info("search_context completed", {
@@ -6138,6 +7361,7 @@ export class RetrievalCore {
 
   private buildEnhancerContextSnippets(results: SearchContextOutput["results"]): EnhancerContextSnippet[] {
     const maxSnippets = this.enhancerGenerationConfig.max_context_snippets;
+    const snippetCharLimit = this.contextPackingConfig.enabled ? this.contextPackingConfig.enhancer_snippet_char_limit : 1_600;
     const snippets: EnhancerContextSnippet[] = [];
     for (const result of results.slice(0, maxSnippets)) {
       snippets.push({
@@ -6145,7 +7369,7 @@ export class RetrievalCore {
         start_line: result.start_line,
         end_line: result.end_line,
         reason: result.reason,
-        snippet: result.snippet.slice(0, 1_600),
+        snippet: result.snippet.slice(0, snippetCharLimit),
         score: result.score
       });
     }
@@ -6320,7 +7544,10 @@ export class RetrievalCore {
             top_k: MAX_TOP_K
           }
         });
-        const budgetedResults = trimToContextBudget(retrieval.results);
+        const budgetedResults = trimToContextBudget(
+          retrieval.results,
+          this.contextPackingConfig.enabled ? "lightweight" : "ranking"
+        );
         const dedupedByPath = dedupeEnhancerCandidatesByPath(budgetedResults);
         const collapsedByDirectory = collapseEnhancerCandidatesByDirectory(
           dedupedByPath,