@rce-mcp/retrieval-core 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +7 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/chunking.d.ts +13 -0
- package/dist/chunking.js +488 -77
- package/dist/index.d.ts +61 -0
- package/dist/index.js +993 -20
- package/dist/remote-sync.js +2 -1
- package/package.json +2 -2
- package/scripts/poc-parser-availability-benchmark.ts +2 -0
- package/src/chunking.ts +573 -80
- package/src/index.ts +1247 -20
- package/src/remote-sync.ts +3 -1
- package/test/benchmark.thresholds.test.ts +8 -0
- package/test/chunking.config.test.ts +47 -1
- package/test/chunking.language-aware.test.ts +227 -0
- package/test/embedding-context-prefix.test.ts +101 -0
- package/test/enhance-confidence.test.ts +4 -4
- package/test/mcp-search-quality.regression.test.ts +691 -4
- package/test/remote-sync.integration.test.ts +5 -1
- package/test/smart-cutoff.config.test.ts +86 -0
- package/test/snippet-integrity.config.test.ts +59 -0
package/dist/index.d.ts
CHANGED
|
@@ -60,6 +60,15 @@ export interface RetrievalRerankConfig {
|
|
|
60
60
|
max_chunks_per_path_file_lookup: number;
|
|
61
61
|
same_directory_penalty: number;
|
|
62
62
|
same_extension_penalty: number;
|
|
63
|
+
merge_overlapping_chunks_enabled: boolean;
|
|
64
|
+
merge_gap_lines: number;
|
|
65
|
+
merge_max_span_lines: number;
|
|
66
|
+
smart_cutoff_enabled: boolean;
|
|
67
|
+
smart_cutoff_min_k: number;
|
|
68
|
+
smart_cutoff_max_k: number;
|
|
69
|
+
smart_cutoff_min_score: number;
|
|
70
|
+
smart_cutoff_top_ratio: number;
|
|
71
|
+
smart_cutoff_delta_abs: number;
|
|
63
72
|
}
|
|
64
73
|
export interface RetrievalScoringConfig {
|
|
65
74
|
candidate_weights: CandidateScoreWeights;
|
|
@@ -83,8 +92,31 @@ export interface RetrievalChunkingConfig {
|
|
|
83
92
|
fallback_strategy: "sliding";
|
|
84
93
|
target_chunk_tokens: number;
|
|
85
94
|
chunk_overlap_tokens: number;
|
|
95
|
+
budget_tokenizer: "ranking" | "lightweight";
|
|
96
|
+
boundary_strictness: "legacy" | "semantic_js_ts";
|
|
86
97
|
parse_timeout_ms: number;
|
|
87
98
|
enabled_languages: string[];
|
|
99
|
+
recursive_semantic_chunking_enabled: boolean;
|
|
100
|
+
semantic_merge_gap_lines: number;
|
|
101
|
+
semantic_merge_max_span_lines: number;
|
|
102
|
+
comment_forward_absorb_enabled: boolean;
|
|
103
|
+
embedding_context_prefix_enabled: boolean;
|
|
104
|
+
}
|
|
105
|
+
export interface RetrievalContextPackingConfig {
|
|
106
|
+
enabled: boolean;
|
|
107
|
+
max_spans_per_result: number;
|
|
108
|
+
max_gap_lines: number;
|
|
109
|
+
max_snippet_chars: number;
|
|
110
|
+
enhancer_snippet_char_limit: number;
|
|
111
|
+
}
|
|
112
|
+
export interface RetrievalSnippetIntegrityConfig {
|
|
113
|
+
enabled: boolean;
|
|
114
|
+
target_languages: string[];
|
|
115
|
+
max_contiguous_gap_lines: number;
|
|
116
|
+
marker_template_version: "v1";
|
|
117
|
+
repair_enabled: boolean;
|
|
118
|
+
repair_max_envelope_lines: number;
|
|
119
|
+
repair_max_snippet_chars: number;
|
|
88
120
|
}
|
|
89
121
|
export type RetrievalScoringConfigInput = Partial<{
|
|
90
122
|
candidate_weights: Partial<CandidateScoreWeights>;
|
|
@@ -98,14 +130,33 @@ export type RetrievalChunkingConfigInput = Partial<{
|
|
|
98
130
|
fallback_strategy: "sliding";
|
|
99
131
|
target_chunk_tokens: number;
|
|
100
132
|
chunk_overlap_tokens: number;
|
|
133
|
+
budget_tokenizer: "ranking" | "lightweight";
|
|
134
|
+
boundary_strictness: "legacy" | "semantic_js_ts";
|
|
101
135
|
parse_timeout_ms: number;
|
|
102
136
|
enabled_languages: string[];
|
|
137
|
+
recursive_semantic_chunking_enabled: boolean;
|
|
138
|
+
semantic_merge_gap_lines: number;
|
|
139
|
+
semantic_merge_max_span_lines: number;
|
|
140
|
+
comment_forward_absorb_enabled: boolean;
|
|
141
|
+
embedding_context_prefix_enabled: boolean;
|
|
142
|
+
}>;
|
|
143
|
+
export type RetrievalContextPackingConfigInput = Partial<RetrievalContextPackingConfig>;
|
|
144
|
+
export type RetrievalSnippetIntegrityConfigInput = Partial<{
|
|
145
|
+
enabled: boolean;
|
|
146
|
+
target_languages: string[];
|
|
147
|
+
max_contiguous_gap_lines: number;
|
|
148
|
+
marker_template_version: "v1";
|
|
149
|
+
repair_enabled: boolean;
|
|
150
|
+
repair_max_envelope_lines: number;
|
|
151
|
+
repair_max_snippet_chars: number;
|
|
103
152
|
}>;
|
|
104
153
|
export declare const BASELINE_RETRIEVAL_SCORING_CONFIG: RetrievalScoringConfig;
|
|
105
154
|
export declare const CONSERVATIVE_RETRIEVAL_SCORING_CONFIG: RetrievalScoringConfig;
|
|
106
155
|
export declare const DEFAULT_RETRIEVAL_ENHANCER_CONFIG: RetrievalEnhancerConfig;
|
|
107
156
|
export declare const DEFAULT_RETRIEVAL_ENHANCER_GENERATION_CONFIG: RetrievalEnhancerGenerationConfig;
|
|
108
157
|
export declare const DEFAULT_RETRIEVAL_CHUNKING_CONFIG: RetrievalChunkingConfig;
|
|
158
|
+
export declare const DEFAULT_RETRIEVAL_CONTEXT_PACKING_CONFIG: RetrievalContextPackingConfig;
|
|
159
|
+
export declare const DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG: RetrievalSnippetIntegrityConfig;
|
|
109
160
|
declare const BUILTIN_RETRIEVAL_SCORING_PROFILES: {
|
|
110
161
|
readonly baseline: RetrievalScoringConfig;
|
|
111
162
|
readonly conservative: RetrievalScoringConfig;
|
|
@@ -119,6 +170,8 @@ export declare function mergeRetrievalScoringConfig(base: RetrievalScoringConfig
|
|
|
119
170
|
export declare function mergeRetrievalEnhancerConfig(base: RetrievalEnhancerConfig, overrides?: RetrievalEnhancerConfigInput): RetrievalEnhancerConfig;
|
|
120
171
|
export declare function mergeRetrievalEnhancerGenerationConfig(base: RetrievalEnhancerGenerationConfig, overrides?: RetrievalEnhancerGenerationConfigInput): RetrievalEnhancerGenerationConfig;
|
|
121
172
|
export declare function mergeRetrievalChunkingConfig(base: RetrievalChunkingConfig, overrides?: RetrievalChunkingConfigInput): RetrievalChunkingConfig;
|
|
173
|
+
export declare function mergeRetrievalContextPackingConfig(base: RetrievalContextPackingConfig, overrides?: RetrievalContextPackingConfigInput): RetrievalContextPackingConfig;
|
|
174
|
+
export declare function mergeRetrievalSnippetIntegrityConfig(base: RetrievalSnippetIntegrityConfig, overrides?: RetrievalSnippetIntegrityConfigInput): RetrievalSnippetIntegrityConfig;
|
|
122
175
|
declare const REASON_STRINGS: readonly ["semantic match", "exact literal match", "path token overlap", "recently modified relevant module"];
|
|
123
176
|
export type RetrievalReason = (typeof REASON_STRINGS)[number];
|
|
124
177
|
export declare class RetrievalError extends Error {
|
|
@@ -203,6 +256,8 @@ export interface RetrievalCoreOptions {
|
|
|
203
256
|
enhancerConfig?: RetrievalEnhancerConfigInput;
|
|
204
257
|
enhancerGenerationConfig?: RetrievalEnhancerGenerationConfigInput;
|
|
205
258
|
chunkingConfig?: RetrievalChunkingConfigInput;
|
|
259
|
+
contextPackingConfig?: RetrievalContextPackingConfigInput;
|
|
260
|
+
snippetIntegrityConfig?: RetrievalSnippetIntegrityConfigInput;
|
|
206
261
|
enhancerDecisionTraceEnabled?: boolean;
|
|
207
262
|
}
|
|
208
263
|
export interface EmbeddingDescriptor {
|
|
@@ -435,6 +490,10 @@ export declare class ClaudeAgentEnhancerProvider implements EnhancerGenerationPr
|
|
|
435
490
|
describe(): EnhancerProviderDescriptor;
|
|
436
491
|
generate(input: EnhancerGenerationRequest): Promise<EnhancerGenerationResult>;
|
|
437
492
|
}
|
|
493
|
+
export declare function __applySmartCutoffCandidatesForTests(input: {
|
|
494
|
+
candidates: SearchContextOutput["results"];
|
|
495
|
+
config: RetrievalRerankConfig;
|
|
496
|
+
}): SearchContextOutput["results"];
|
|
438
497
|
export declare class InMemoryIndexStore implements IndexRepository {
|
|
439
498
|
private readonly workspaces;
|
|
440
499
|
private readonly workspacesByPath;
|
|
@@ -569,6 +628,8 @@ export declare class RetrievalCore {
|
|
|
569
628
|
private readonly enhancerConfig;
|
|
570
629
|
private readonly enhancerGenerationConfig;
|
|
571
630
|
private readonly chunkingConfig;
|
|
631
|
+
private readonly contextPackingConfig;
|
|
632
|
+
private readonly snippetIntegrityConfig;
|
|
572
633
|
private readonly enhancerDecisionTraceEnabled;
|
|
573
634
|
private cacheHits;
|
|
574
635
|
private cacheMisses;
|