@rce-mcp/retrieval-core 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/chunking.d.ts +13 -0
- package/dist/chunking.js +493 -81
- package/dist/index.d.ts +280 -4
- package/dist/index.js +2960 -235
- package/dist/remote-sync.js +4 -2
- package/package.json +8 -6
- package/scripts/poc-parser-availability-benchmark.ts +2 -0
- package/src/chunking.ts +578 -84
- package/src/index.ts +3818 -401
- package/src/remote-sync.ts +6 -2
- package/test/benchmark.thresholds.test.ts +63 -0
- package/test/chunking.config.test.ts +74 -0
- package/test/chunking.language-aware.test.ts +250 -4
- package/test/chunking.parser-availability.poc.test.ts +3 -3
- package/test/claude-agent-provider.test.ts +209 -0
- package/test/embedding-context-prefix.test.ts +101 -0
- package/test/embedding-provider.test.ts +450 -1
- package/test/enhance-confidence.test.ts +275 -3
- package/test/integration.test.ts +185 -1
- package/test/mcp-search-quality.regression.test.ts +1009 -0
- package/test/remote-sync.integration.test.ts +15 -0
- package/test/smart-cutoff.config.test.ts +86 -0
- package/test/snippet-integrity.config.test.ts +59 -0
package/dist/index.js
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import { buildQueryCacheKey } from "@rce-mcp/data-plane";
|
|
2
|
+
import { buildQueryCacheKey, tokenizeForRanking } from "@rce-mcp/data-plane";
|
|
3
3
|
import { InMemoryQueryCache } from "@rce-mcp/data-plane";
|
|
4
4
|
import { getObservability } from "@rce-mcp/observability";
|
|
5
5
|
import { buildChunksForFile, getChunkingParserAvailabilitySnapshot } from "./chunking.js";
|
|
6
6
|
const MAX_FILE_SIZE_BYTES = 1_000_000;
|
|
7
7
|
const MAX_CHUNKS_PER_FILE = 300;
|
|
8
|
-
const
|
|
9
|
-
const
|
|
8
|
+
const DEFAULT_TARGET_CHUNK_TOKENS = 420;
|
|
9
|
+
const DEFAULT_CHUNK_OVERLAP_TOKENS = 90;
|
|
10
10
|
const MAX_TOP_K = 20;
|
|
11
|
+
const DEFAULT_INTERNAL_CANDIDATE_DEPTH = 100;
|
|
12
|
+
const MIN_INTERNAL_CANDIDATE_DEPTH = 20;
|
|
13
|
+
const MAX_INTERNAL_CANDIDATE_DEPTH = 200;
|
|
11
14
|
const MAX_CONTEXT_BUDGET_TOKENS = 12_000;
|
|
12
15
|
export const DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_BASE_URL = "https://router.tumuer.me/v1";
|
|
13
16
|
export const DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_MODEL = "Qwen/Qwen3-Embedding-4B";
|
|
@@ -15,6 +18,17 @@ export const DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_DIMENSIONS = 2560;
|
|
|
15
18
|
export const DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_TIMEOUT_MS = 10_000;
|
|
16
19
|
export const DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_BATCH_SIZE = 64;
|
|
17
20
|
export const DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_MAX_RETRIES = 2;
|
|
21
|
+
export const DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_TRANSIENT_403_MAX_RETRIES = 4;
|
|
22
|
+
export const DEFAULT_OPENAI_COMPATIBLE_RERANKER_BASE_URL = "https://router.tumuer.me/v1";
|
|
23
|
+
export const DEFAULT_OPENAI_COMPATIBLE_RERANKER_MODEL = "Qwen/Qwen3-Reranker-4B";
|
|
24
|
+
export const DEFAULT_OPENAI_COMPATIBLE_RERANKER_TIMEOUT_MS = 2_500;
|
|
25
|
+
export const DEFAULT_SEARCH_RERANKER_TOP_N = 30;
|
|
26
|
+
export const DEFAULT_PROVIDER_MAX_REQUESTS_PER_MINUTE = 90;
|
|
27
|
+
export const DEFAULT_PROVIDER_LIMIT_INDEX_MAX_WAIT_MS = 120_000;
|
|
28
|
+
export const DEFAULT_PROVIDER_LIMIT_QUERY_MAX_WAIT_MS = 1_000;
|
|
29
|
+
export const DEFAULT_PROVIDER_LIMIT_RERANK_MAX_WAIT_MS = 500;
|
|
30
|
+
export const DEFAULT_CLAUDE_ENHANCER_MODEL = "claude-3-5-sonnet-latest";
|
|
31
|
+
const DEFAULT_CLAUDE_ENHANCER_MAX_TURNS = 3;
|
|
18
32
|
const DEFAULT_CANDIDATE_SCORE_WEIGHTS = {
|
|
19
33
|
lexical_weight: 0.6,
|
|
20
34
|
vector_weight: 0.4,
|
|
@@ -52,6 +66,10 @@ export const BASELINE_RETRIEVAL_SCORING_CONFIG = {
|
|
|
52
66
|
negation_avoid_tests_penalty: 0.35,
|
|
53
67
|
negation_avoid_examples_penalty: 0.3,
|
|
54
68
|
negation_avoid_archive_penalty: 0.35,
|
|
69
|
+
security_trace_meta_penalty: 0.22,
|
|
70
|
+
literal_path_boost: 0.3,
|
|
71
|
+
literal_snippet_boost: 0.18,
|
|
72
|
+
literal_max_boost: 0.5,
|
|
55
73
|
min_total_bias: -0.45,
|
|
56
74
|
max_total_bias: 0.35
|
|
57
75
|
},
|
|
@@ -60,7 +78,16 @@ export const BASELINE_RETRIEVAL_SCORING_CONFIG = {
|
|
|
60
78
|
max_chunks_per_path_default: 2,
|
|
61
79
|
max_chunks_per_path_file_lookup: 1,
|
|
62
80
|
same_directory_penalty: 0,
|
|
63
|
-
same_extension_penalty: 0
|
|
81
|
+
same_extension_penalty: 0,
|
|
82
|
+
merge_overlapping_chunks_enabled: true,
|
|
83
|
+
merge_gap_lines: 6,
|
|
84
|
+
merge_max_span_lines: 220,
|
|
85
|
+
smart_cutoff_enabled: false,
|
|
86
|
+
smart_cutoff_min_k: 2,
|
|
87
|
+
smart_cutoff_max_k: 8,
|
|
88
|
+
smart_cutoff_min_score: 0.25,
|
|
89
|
+
smart_cutoff_top_ratio: 0.5,
|
|
90
|
+
smart_cutoff_delta_abs: 0.25
|
|
64
91
|
}
|
|
65
92
|
};
|
|
66
93
|
export const CONSERVATIVE_RETRIEVAL_SCORING_CONFIG = {
|
|
@@ -99,6 +126,10 @@ export const CONSERVATIVE_RETRIEVAL_SCORING_CONFIG = {
|
|
|
99
126
|
negation_avoid_tests_penalty: 0.2,
|
|
100
127
|
negation_avoid_examples_penalty: 0.16,
|
|
101
128
|
negation_avoid_archive_penalty: 0.2,
|
|
129
|
+
security_trace_meta_penalty: 0.14,
|
|
130
|
+
literal_path_boost: 0.18,
|
|
131
|
+
literal_snippet_boost: 0.1,
|
|
132
|
+
literal_max_boost: 0.28,
|
|
102
133
|
min_total_bias: -0.25,
|
|
103
134
|
max_total_bias: 0.2
|
|
104
135
|
},
|
|
@@ -107,7 +138,16 @@ export const CONSERVATIVE_RETRIEVAL_SCORING_CONFIG = {
|
|
|
107
138
|
max_chunks_per_path_default: 2,
|
|
108
139
|
max_chunks_per_path_file_lookup: 1,
|
|
109
140
|
same_directory_penalty: 0,
|
|
110
|
-
same_extension_penalty: 0
|
|
141
|
+
same_extension_penalty: 0,
|
|
142
|
+
merge_overlapping_chunks_enabled: true,
|
|
143
|
+
merge_gap_lines: 6,
|
|
144
|
+
merge_max_span_lines: 220,
|
|
145
|
+
smart_cutoff_enabled: false,
|
|
146
|
+
smart_cutoff_min_k: 2,
|
|
147
|
+
smart_cutoff_max_k: 8,
|
|
148
|
+
smart_cutoff_min_score: 0.25,
|
|
149
|
+
smart_cutoff_top_ratio: 0.5,
|
|
150
|
+
smart_cutoff_delta_abs: 0.25
|
|
111
151
|
}
|
|
112
152
|
};
|
|
113
153
|
export const DEFAULT_RETRIEVAL_ENHANCER_CONFIG = {
|
|
@@ -115,11 +155,42 @@ export const DEFAULT_RETRIEVAL_ENHANCER_CONFIG = {
|
|
|
115
155
|
max_candidates_pre_rerank: 4,
|
|
116
156
|
rerank_timeout_ms: 40
|
|
117
157
|
};
|
|
158
|
+
export const DEFAULT_RETRIEVAL_ENHANCER_GENERATION_CONFIG = {
|
|
159
|
+
timeout_ms: 18_000,
|
|
160
|
+
max_retries: 1,
|
|
161
|
+
tool_mode: "read_only",
|
|
162
|
+
max_context_snippets: 6
|
|
163
|
+
};
|
|
118
164
|
export const DEFAULT_RETRIEVAL_CHUNKING_CONFIG = {
|
|
119
165
|
strategy: "sliding",
|
|
120
166
|
fallback_strategy: "sliding",
|
|
167
|
+
target_chunk_tokens: DEFAULT_TARGET_CHUNK_TOKENS,
|
|
168
|
+
chunk_overlap_tokens: DEFAULT_CHUNK_OVERLAP_TOKENS,
|
|
169
|
+
budget_tokenizer: "ranking",
|
|
170
|
+
boundary_strictness: "legacy",
|
|
121
171
|
parse_timeout_ms: 80,
|
|
122
|
-
enabled_languages: ["typescript", "javascript", "python", "go"]
|
|
172
|
+
enabled_languages: ["typescript", "javascript", "python", "go"],
|
|
173
|
+
recursive_semantic_chunking_enabled: false,
|
|
174
|
+
semantic_merge_gap_lines: 6,
|
|
175
|
+
semantic_merge_max_span_lines: 220,
|
|
176
|
+
comment_forward_absorb_enabled: true,
|
|
177
|
+
embedding_context_prefix_enabled: true
|
|
178
|
+
};
|
|
179
|
+
export const DEFAULT_RETRIEVAL_CONTEXT_PACKING_CONFIG = {
|
|
180
|
+
enabled: false,
|
|
181
|
+
max_spans_per_result: 3,
|
|
182
|
+
max_gap_lines: 120,
|
|
183
|
+
max_snippet_chars: 3_200,
|
|
184
|
+
enhancer_snippet_char_limit: 2_200
|
|
185
|
+
};
|
|
186
|
+
export const DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG = {
|
|
187
|
+
enabled: false,
|
|
188
|
+
target_languages: ["typescript", "tsx", "javascript", "jsx"],
|
|
189
|
+
max_contiguous_gap_lines: 6,
|
|
190
|
+
marker_template_version: "v1",
|
|
191
|
+
repair_enabled: false,
|
|
192
|
+
repair_max_envelope_lines: 260,
|
|
193
|
+
repair_max_snippet_chars: 3_600
|
|
123
194
|
};
|
|
124
195
|
const BUILTIN_RETRIEVAL_SCORING_PROFILES = {
|
|
125
196
|
baseline: BASELINE_RETRIEVAL_SCORING_CONFIG,
|
|
@@ -172,6 +243,36 @@ function validateScoringConfig(config) {
|
|
|
172
243
|
if (rerank.same_extension_penalty < 0) {
|
|
173
244
|
throw new Error("invalid retrieval scoring config: rerank.same_extension_penalty must be >= 0");
|
|
174
245
|
}
|
|
246
|
+
if (typeof rerank.merge_overlapping_chunks_enabled !== "boolean") {
|
|
247
|
+
throw new Error("invalid retrieval scoring config: rerank.merge_overlapping_chunks_enabled must be boolean");
|
|
248
|
+
}
|
|
249
|
+
if (!Number.isInteger(rerank.merge_gap_lines) || rerank.merge_gap_lines < 0) {
|
|
250
|
+
throw new Error("invalid retrieval scoring config: rerank.merge_gap_lines must be an integer >= 0");
|
|
251
|
+
}
|
|
252
|
+
if (!Number.isInteger(rerank.merge_max_span_lines) || rerank.merge_max_span_lines <= 0) {
|
|
253
|
+
throw new Error("invalid retrieval scoring config: rerank.merge_max_span_lines must be a positive integer");
|
|
254
|
+
}
|
|
255
|
+
if (typeof rerank.smart_cutoff_enabled !== "boolean") {
|
|
256
|
+
throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_enabled must be boolean");
|
|
257
|
+
}
|
|
258
|
+
if (!Number.isInteger(rerank.smart_cutoff_min_k) || rerank.smart_cutoff_min_k <= 0) {
|
|
259
|
+
throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_min_k must be a positive integer");
|
|
260
|
+
}
|
|
261
|
+
if (!Number.isInteger(rerank.smart_cutoff_max_k) || rerank.smart_cutoff_max_k <= 0) {
|
|
262
|
+
throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_max_k must be a positive integer");
|
|
263
|
+
}
|
|
264
|
+
if (rerank.smart_cutoff_max_k < rerank.smart_cutoff_min_k) {
|
|
265
|
+
throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_max_k must be >= smart_cutoff_min_k");
|
|
266
|
+
}
|
|
267
|
+
assertFiniteNumber(rerank.smart_cutoff_min_score, "rerank.smart_cutoff_min_score");
|
|
268
|
+
assertFiniteNumber(rerank.smart_cutoff_top_ratio, "rerank.smart_cutoff_top_ratio");
|
|
269
|
+
assertFiniteNumber(rerank.smart_cutoff_delta_abs, "rerank.smart_cutoff_delta_abs");
|
|
270
|
+
if (rerank.smart_cutoff_top_ratio <= 0 || rerank.smart_cutoff_top_ratio > 1) {
|
|
271
|
+
throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_top_ratio must be in (0, 1]");
|
|
272
|
+
}
|
|
273
|
+
if (rerank.smart_cutoff_delta_abs < 0) {
|
|
274
|
+
throw new Error("invalid retrieval scoring config: rerank.smart_cutoff_delta_abs must be >= 0");
|
|
275
|
+
}
|
|
175
276
|
}
|
|
176
277
|
export function resolveRetrievalScoringProfile(profile_id) {
|
|
177
278
|
const normalized = (profile_id ?? "baseline").trim().toLowerCase();
|
|
@@ -225,6 +326,28 @@ export function mergeRetrievalEnhancerConfig(base, overrides) {
|
|
|
225
326
|
validateEnhancerConfig(next);
|
|
226
327
|
return next;
|
|
227
328
|
}
|
|
329
|
+
function validateEnhancerGenerationConfig(config) {
|
|
330
|
+
if (!Number.isInteger(config.timeout_ms) || config.timeout_ms <= 0) {
|
|
331
|
+
throw new Error("invalid retrieval enhancer generation config: timeout_ms must be a positive integer");
|
|
332
|
+
}
|
|
333
|
+
if (!Number.isInteger(config.max_retries) || config.max_retries < 0) {
|
|
334
|
+
throw new Error("invalid retrieval enhancer generation config: max_retries must be a non-negative integer");
|
|
335
|
+
}
|
|
336
|
+
if (config.tool_mode !== "none" && config.tool_mode !== "read_only") {
|
|
337
|
+
throw new Error("invalid retrieval enhancer generation config: tool_mode must be none|read_only");
|
|
338
|
+
}
|
|
339
|
+
if (!Number.isInteger(config.max_context_snippets) || config.max_context_snippets <= 0) {
|
|
340
|
+
throw new Error("invalid retrieval enhancer generation config: max_context_snippets must be a positive integer");
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
export function mergeRetrievalEnhancerGenerationConfig(base, overrides) {
|
|
344
|
+
const next = {
|
|
345
|
+
...base,
|
|
346
|
+
...(overrides ?? {})
|
|
347
|
+
};
|
|
348
|
+
validateEnhancerGenerationConfig(next);
|
|
349
|
+
return next;
|
|
350
|
+
}
|
|
228
351
|
function normalizeChunkingLanguageList(value) {
|
|
229
352
|
const deduped = new Set();
|
|
230
353
|
for (const language of value) {
|
|
@@ -243,9 +366,24 @@ function validateChunkingConfig(config) {
|
|
|
243
366
|
if (config.fallback_strategy !== "sliding") {
|
|
244
367
|
throw new Error("invalid retrieval chunking config: fallback_strategy must be sliding");
|
|
245
368
|
}
|
|
369
|
+
if (!Number.isInteger(config.target_chunk_tokens) || config.target_chunk_tokens <= 0) {
|
|
370
|
+
throw new Error("invalid retrieval chunking config: target_chunk_tokens must be a positive integer");
|
|
371
|
+
}
|
|
372
|
+
if (!Number.isInteger(config.chunk_overlap_tokens) || config.chunk_overlap_tokens <= 0) {
|
|
373
|
+
throw new Error("invalid retrieval chunking config: chunk_overlap_tokens must be a positive integer");
|
|
374
|
+
}
|
|
375
|
+
if (config.chunk_overlap_tokens >= config.target_chunk_tokens) {
|
|
376
|
+
throw new Error("invalid retrieval chunking config: chunk_overlap_tokens must be less than target_chunk_tokens");
|
|
377
|
+
}
|
|
246
378
|
if (!Number.isInteger(config.parse_timeout_ms) || config.parse_timeout_ms <= 0) {
|
|
247
379
|
throw new Error("invalid retrieval chunking config: parse_timeout_ms must be a positive integer");
|
|
248
380
|
}
|
|
381
|
+
if (config.budget_tokenizer !== "ranking" && config.budget_tokenizer !== "lightweight") {
|
|
382
|
+
throw new Error("invalid retrieval chunking config: budget_tokenizer must be ranking|lightweight");
|
|
383
|
+
}
|
|
384
|
+
if (config.boundary_strictness !== "legacy" && config.boundary_strictness !== "semantic_js_ts") {
|
|
385
|
+
throw new Error("invalid retrieval chunking config: boundary_strictness must be legacy|semantic_js_ts");
|
|
386
|
+
}
|
|
249
387
|
if (!Array.isArray(config.enabled_languages) || config.enabled_languages.length === 0) {
|
|
250
388
|
throw new Error("invalid retrieval chunking config: enabled_languages must include at least one language");
|
|
251
389
|
}
|
|
@@ -254,6 +392,21 @@ function validateChunkingConfig(config) {
|
|
|
254
392
|
throw new Error("invalid retrieval chunking config: enabled_languages must contain non-empty strings");
|
|
255
393
|
}
|
|
256
394
|
}
|
|
395
|
+
if (typeof config.recursive_semantic_chunking_enabled !== "boolean") {
|
|
396
|
+
throw new Error("invalid retrieval chunking config: recursive_semantic_chunking_enabled must be boolean");
|
|
397
|
+
}
|
|
398
|
+
if (!Number.isInteger(config.semantic_merge_gap_lines) || config.semantic_merge_gap_lines < 0) {
|
|
399
|
+
throw new Error("invalid retrieval chunking config: semantic_merge_gap_lines must be a non-negative integer");
|
|
400
|
+
}
|
|
401
|
+
if (!Number.isInteger(config.semantic_merge_max_span_lines) || config.semantic_merge_max_span_lines <= 0) {
|
|
402
|
+
throw new Error("invalid retrieval chunking config: semantic_merge_max_span_lines must be a positive integer");
|
|
403
|
+
}
|
|
404
|
+
if (typeof config.comment_forward_absorb_enabled !== "boolean") {
|
|
405
|
+
throw new Error("invalid retrieval chunking config: comment_forward_absorb_enabled must be boolean");
|
|
406
|
+
}
|
|
407
|
+
if (typeof config.embedding_context_prefix_enabled !== "boolean") {
|
|
408
|
+
throw new Error("invalid retrieval chunking config: embedding_context_prefix_enabled must be boolean");
|
|
409
|
+
}
|
|
257
410
|
}
|
|
258
411
|
export function mergeRetrievalChunkingConfig(base, overrides) {
|
|
259
412
|
const next = {
|
|
@@ -264,6 +417,98 @@ export function mergeRetrievalChunkingConfig(base, overrides) {
|
|
|
264
417
|
validateChunkingConfig(next);
|
|
265
418
|
return next;
|
|
266
419
|
}
|
|
420
|
+
function validateContextPackingConfig(config) {
|
|
421
|
+
if (typeof config.enabled !== "boolean") {
|
|
422
|
+
throw new Error("invalid retrieval context packing config: enabled must be boolean");
|
|
423
|
+
}
|
|
424
|
+
if (!Number.isInteger(config.max_spans_per_result) || config.max_spans_per_result <= 0) {
|
|
425
|
+
throw new Error("invalid retrieval context packing config: max_spans_per_result must be a positive integer");
|
|
426
|
+
}
|
|
427
|
+
if (!Number.isInteger(config.max_gap_lines) || config.max_gap_lines < 0) {
|
|
428
|
+
throw new Error("invalid retrieval context packing config: max_gap_lines must be a non-negative integer");
|
|
429
|
+
}
|
|
430
|
+
if (!Number.isInteger(config.max_snippet_chars) || config.max_snippet_chars <= 0) {
|
|
431
|
+
throw new Error("invalid retrieval context packing config: max_snippet_chars must be a positive integer");
|
|
432
|
+
}
|
|
433
|
+
if (!Number.isInteger(config.enhancer_snippet_char_limit) || config.enhancer_snippet_char_limit <= 0) {
|
|
434
|
+
throw new Error("invalid retrieval context packing config: enhancer_snippet_char_limit must be a positive integer");
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
export function mergeRetrievalContextPackingConfig(base, overrides) {
|
|
438
|
+
const next = {
|
|
439
|
+
...base,
|
|
440
|
+
...(overrides ?? {})
|
|
441
|
+
};
|
|
442
|
+
validateContextPackingConfig(next);
|
|
443
|
+
return next;
|
|
444
|
+
}
|
|
445
|
+
function normalizeSnippetIntegrityLanguage(value) {
|
|
446
|
+
const normalized = value.trim().toLowerCase();
|
|
447
|
+
if (normalized === "typescript" || normalized === "ts" || normalized === "mts" || normalized === "cts") {
|
|
448
|
+
return "typescript";
|
|
449
|
+
}
|
|
450
|
+
if (normalized === "tsx") {
|
|
451
|
+
return "tsx";
|
|
452
|
+
}
|
|
453
|
+
if (normalized === "javascript" || normalized === "js" || normalized === "mjs" || normalized === "cjs") {
|
|
454
|
+
return "javascript";
|
|
455
|
+
}
|
|
456
|
+
if (normalized === "jsx") {
|
|
457
|
+
return "jsx";
|
|
458
|
+
}
|
|
459
|
+
return undefined;
|
|
460
|
+
}
|
|
461
|
+
function normalizeSnippetIntegrityLanguageList(value) {
|
|
462
|
+
const deduped = new Set();
|
|
463
|
+
for (const language of value) {
|
|
464
|
+
const raw = language.trim().toLowerCase();
|
|
465
|
+
if (raw.length === 0) {
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
deduped.add(normalizeSnippetIntegrityLanguage(raw) ?? raw);
|
|
469
|
+
}
|
|
470
|
+
return [...deduped];
|
|
471
|
+
}
|
|
472
|
+
function validateSnippetIntegrityConfig(config) {
|
|
473
|
+
if (typeof config.enabled !== "boolean") {
|
|
474
|
+
throw new Error("invalid retrieval snippet integrity config: enabled must be boolean");
|
|
475
|
+
}
|
|
476
|
+
if (!Array.isArray(config.target_languages) || config.target_languages.length === 0) {
|
|
477
|
+
throw new Error("invalid retrieval snippet integrity config: target_languages must include at least one language");
|
|
478
|
+
}
|
|
479
|
+
for (const language of config.target_languages) {
|
|
480
|
+
if (typeof language !== "string" || language.trim().length === 0) {
|
|
481
|
+
throw new Error("invalid retrieval snippet integrity config: target_languages must contain non-empty strings");
|
|
482
|
+
}
|
|
483
|
+
if (!normalizeSnippetIntegrityLanguage(language)) {
|
|
484
|
+
throw new Error("invalid retrieval snippet integrity config: unsupported target language");
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
if (!Number.isInteger(config.max_contiguous_gap_lines) || config.max_contiguous_gap_lines < 0) {
|
|
488
|
+
throw new Error("invalid retrieval snippet integrity config: max_contiguous_gap_lines must be a non-negative integer");
|
|
489
|
+
}
|
|
490
|
+
if (config.marker_template_version !== "v1") {
|
|
491
|
+
throw new Error("invalid retrieval snippet integrity config: marker_template_version must be v1");
|
|
492
|
+
}
|
|
493
|
+
if (typeof config.repair_enabled !== "boolean") {
|
|
494
|
+
throw new Error("invalid retrieval snippet integrity config: repair_enabled must be boolean");
|
|
495
|
+
}
|
|
496
|
+
if (!Number.isInteger(config.repair_max_envelope_lines) || config.repair_max_envelope_lines <= 0) {
|
|
497
|
+
throw new Error("invalid retrieval snippet integrity config: repair_max_envelope_lines must be a positive integer");
|
|
498
|
+
}
|
|
499
|
+
if (!Number.isInteger(config.repair_max_snippet_chars) || config.repair_max_snippet_chars <= 0) {
|
|
500
|
+
throw new Error("invalid retrieval snippet integrity config: repair_max_snippet_chars must be a positive integer");
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
export function mergeRetrievalSnippetIntegrityConfig(base, overrides) {
|
|
504
|
+
const next = {
|
|
505
|
+
...base,
|
|
506
|
+
...(overrides ?? {}),
|
|
507
|
+
target_languages: normalizeSnippetIntegrityLanguageList(overrides?.target_languages ?? base.target_languages)
|
|
508
|
+
};
|
|
509
|
+
validateSnippetIntegrityConfig(next);
|
|
510
|
+
return next;
|
|
511
|
+
}
|
|
267
512
|
function stableSerialize(value) {
|
|
268
513
|
if (Array.isArray(value)) {
|
|
269
514
|
return `[${value.map((entry) => stableSerialize(entry)).join(",")}]`;
|
|
@@ -277,10 +522,14 @@ function stableSerialize(value) {
|
|
|
277
522
|
function scoringConfigChecksum(config) {
|
|
278
523
|
return sha256(stableSerialize(config)).slice(0, 12);
|
|
279
524
|
}
|
|
525
|
+
function clampInternalCandidateDepth(value) {
|
|
526
|
+
const raw = Number.isFinite(value) ? Math.trunc(value ?? DEFAULT_INTERNAL_CANDIDATE_DEPTH) : DEFAULT_INTERNAL_CANDIDATE_DEPTH;
|
|
527
|
+
return Math.max(MIN_INTERNAL_CANDIDATE_DEPTH, Math.min(MAX_INTERNAL_CANDIDATE_DEPTH, raw));
|
|
528
|
+
}
|
|
280
529
|
const REASON_STRINGS = [
|
|
281
530
|
"semantic match",
|
|
282
|
-
"exact
|
|
283
|
-
"path
|
|
531
|
+
"exact literal match",
|
|
532
|
+
"path token overlap",
|
|
284
533
|
"recently modified relevant module"
|
|
285
534
|
];
|
|
286
535
|
export class RetrievalError extends Error {
|
|
@@ -293,10 +542,26 @@ export class RetrievalError extends Error {
|
|
|
293
542
|
class EmbeddingProviderRequestError extends Error {
|
|
294
543
|
reason;
|
|
295
544
|
retryable;
|
|
296
|
-
|
|
545
|
+
retry_after_ms;
|
|
546
|
+
constructor(reason, retryable, message, retry_after_ms) {
|
|
297
547
|
super(message);
|
|
298
548
|
this.reason = reason;
|
|
299
549
|
this.retryable = retryable;
|
|
550
|
+
this.retry_after_ms = retry_after_ms;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
class RerankerProviderRequestError extends Error {
|
|
554
|
+
reason;
|
|
555
|
+
constructor(reason, message) {
|
|
556
|
+
super(message);
|
|
557
|
+
this.reason = reason;
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
class EnhancerProviderRequestError extends Error {
|
|
561
|
+
reason;
|
|
562
|
+
constructor(reason, message) {
|
|
563
|
+
super(message);
|
|
564
|
+
this.reason = reason;
|
|
300
565
|
}
|
|
301
566
|
}
|
|
302
567
|
const SECRET_PATTERNS = [
|
|
@@ -341,42 +606,21 @@ function singularizeToken(token) {
|
|
|
341
606
|
}
|
|
342
607
|
return undefined;
|
|
343
608
|
}
|
|
344
|
-
function
|
|
345
|
-
|
|
346
|
-
.
|
|
347
|
-
.
|
|
609
|
+
function tokenizeLightweight(text) {
|
|
610
|
+
return text
|
|
611
|
+
.normalize("NFKC")
|
|
612
|
+
.split(/[^A-Za-z0-9_]+/)
|
|
613
|
+
.map((token) => token.trim().toLowerCase())
|
|
348
614
|
.filter(Boolean);
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
const singular = singularizeToken(normalized);
|
|
357
|
-
if (singular) {
|
|
358
|
-
expandedTokens.add(singular);
|
|
359
|
-
}
|
|
360
|
-
const plural = pluralizeToken(normalized);
|
|
361
|
-
if (plural) {
|
|
362
|
-
expandedTokens.add(plural);
|
|
363
|
-
}
|
|
364
|
-
};
|
|
365
|
-
for (const token of coarseTokens) {
|
|
366
|
-
addToken(token);
|
|
367
|
-
for (const part of token.split(/[./_-]+/).filter(Boolean)) {
|
|
368
|
-
addToken(part);
|
|
369
|
-
const camelSplit = part
|
|
370
|
-
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
371
|
-
.split(/\s+/)
|
|
372
|
-
.map((segment) => segment.trim().toLowerCase())
|
|
373
|
-
.filter(Boolean);
|
|
374
|
-
for (const segment of camelSplit) {
|
|
375
|
-
addToken(segment);
|
|
376
|
-
}
|
|
377
|
-
}
|
|
615
|
+
}
|
|
616
|
+
function tokenize(text) {
|
|
617
|
+
return tokenizeForRanking(text);
|
|
618
|
+
}
|
|
619
|
+
function chunkBudgetTokenize(text, mode) {
|
|
620
|
+
if (mode === "lightweight") {
|
|
621
|
+
return tokenizeLightweight(text);
|
|
378
622
|
}
|
|
379
|
-
return
|
|
623
|
+
return tokenize(text);
|
|
380
624
|
}
|
|
381
625
|
function lexicalScore(query, haystack) {
|
|
382
626
|
const q = new Set(tokenize(query));
|
|
@@ -419,17 +663,101 @@ function looksLowInformation(snippet) {
|
|
|
419
663
|
return /^(.)\1+$/.test(noWhitespace);
|
|
420
664
|
}
|
|
421
665
|
function chooseReason(input) {
|
|
666
|
+
if (input.literal_match) {
|
|
667
|
+
return "exact literal match";
|
|
668
|
+
}
|
|
422
669
|
if (input.path_match) {
|
|
423
|
-
return "
|
|
670
|
+
return "path token overlap";
|
|
424
671
|
}
|
|
425
672
|
if (input.recency_boosted) {
|
|
426
673
|
return "recently modified relevant module";
|
|
427
674
|
}
|
|
428
675
|
if (input.lexical > 0.3) {
|
|
429
|
-
return "path
|
|
676
|
+
return "path token overlap";
|
|
430
677
|
}
|
|
431
678
|
return "semantic match";
|
|
432
679
|
}
|
|
680
|
+
function isExactLiteralReason(reason) {
|
|
681
|
+
return reason === "exact literal match" || reason === "exact symbol match";
|
|
682
|
+
}
|
|
683
|
+
function extractSearchLiterals(query) {
|
|
684
|
+
const literals = [];
|
|
685
|
+
const seen = new Set();
|
|
686
|
+
const addLiteral = (raw) => {
|
|
687
|
+
const cleaned = raw.trim().replace(/^[`"'([{]+|[`"')\]}:;,.]+$/g, "");
|
|
688
|
+
const normalized = cleaned.toLowerCase();
|
|
689
|
+
if (!normalized || seen.has(normalized)) {
|
|
690
|
+
return;
|
|
691
|
+
}
|
|
692
|
+
if (normalized.length < 3) {
|
|
693
|
+
return;
|
|
694
|
+
}
|
|
695
|
+
const looksEnvLiteral = /^[A-Z0-9]+(?:_[A-Z0-9]+){2,}$/.test(cleaned);
|
|
696
|
+
const looksPathOrFileLiteral = /[/.]/.test(cleaned);
|
|
697
|
+
const looksCamelLiteral = /[a-z][A-Z]/.test(cleaned) || /[A-Z][a-z]+[A-Z]/.test(cleaned);
|
|
698
|
+
const looksHyphenLiteral = cleaned.includes("-");
|
|
699
|
+
const isSpecificLiteral = looksEnvLiteral || looksPathOrFileLiteral || looksCamelLiteral || looksHyphenLiteral;
|
|
700
|
+
if (!isSpecificLiteral) {
|
|
701
|
+
return;
|
|
702
|
+
}
|
|
703
|
+
seen.add(normalized);
|
|
704
|
+
literals.push(normalized);
|
|
705
|
+
};
|
|
706
|
+
for (const symbol of extractLikelyCodeSymbols(query, 24)) {
|
|
707
|
+
addLiteral(symbol);
|
|
708
|
+
}
|
|
709
|
+
for (const pathSymbol of extractPathLikeSymbols(query)) {
|
|
710
|
+
addLiteral(pathSymbol);
|
|
711
|
+
const leaf = normalizePath(pathSymbol).split("/").pop();
|
|
712
|
+
if (leaf) {
|
|
713
|
+
addLiteral(leaf);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
for (const envMatch of query.matchAll(/\bRCE_[A-Z0-9_]{4,}\b/g)) {
|
|
717
|
+
addLiteral(envMatch[0] ?? "");
|
|
718
|
+
}
|
|
719
|
+
for (const fileName of query.matchAll(/\b[A-Za-z0-9_.-]+\.(?:ts|tsx|js|jsx|mjs|cjs|py|go|json|md)\b/g)) {
|
|
720
|
+
addLiteral(fileName[0] ?? "");
|
|
721
|
+
}
|
|
722
|
+
return literals.slice(0, 24);
|
|
723
|
+
}
|
|
724
|
+
function applyLiteralBoost(input) {
|
|
725
|
+
if (input.literals.length === 0) {
|
|
726
|
+
return {
|
|
727
|
+
boost: 0,
|
|
728
|
+
matched: false,
|
|
729
|
+
path_matches: 0,
|
|
730
|
+
snippet_matches: 0
|
|
731
|
+
};
|
|
732
|
+
}
|
|
733
|
+
const normalizedPath = input.path.toLowerCase();
|
|
734
|
+
const normalizedSnippet = input.snippet.toLowerCase();
|
|
735
|
+
const pathBias = input.path_bias;
|
|
736
|
+
let boost = 0;
|
|
737
|
+
let pathMatches = 0;
|
|
738
|
+
let snippetMatches = 0;
|
|
739
|
+
for (const literal of input.literals) {
|
|
740
|
+
if (normalizedPath.includes(literal)) {
|
|
741
|
+
boost += pathBias.literal_path_boost;
|
|
742
|
+
pathMatches += 1;
|
|
743
|
+
continue;
|
|
744
|
+
}
|
|
745
|
+
if (normalizedSnippet.includes(literal)) {
|
|
746
|
+
boost += pathBias.literal_snippet_boost;
|
|
747
|
+
snippetMatches += 1;
|
|
748
|
+
}
|
|
749
|
+
if (boost >= pathBias.literal_max_boost) {
|
|
750
|
+
break;
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
const clampedBoost = Math.min(pathBias.literal_max_boost, boost);
|
|
754
|
+
return {
|
|
755
|
+
boost: clampedBoost,
|
|
756
|
+
matched: clampedBoost > 0,
|
|
757
|
+
path_matches: pathMatches,
|
|
758
|
+
snippet_matches: snippetMatches
|
|
759
|
+
};
|
|
760
|
+
}
|
|
433
761
|
const DOC_INTENT_TOKENS = new Set([
|
|
434
762
|
"adr",
|
|
435
763
|
"architecture",
|
|
@@ -483,6 +811,22 @@ const UI_COMPONENT_TOKENS = new Set(["component", "layout", "react", "tsx", "ui"
|
|
|
483
811
|
const FILE_LOOKUP_TOKENS = new Set(["entrypoint", "file", "locate", "path", "where", "which"]);
|
|
484
812
|
const TEST_INTENT_TOKENS = new Set(["assert", "coverage", "e2e", "integration", "spec", "test", "tests", "unit"]);
|
|
485
813
|
const EXAMPLE_INTENT_TOKENS = new Set(["demo", "example", "examples", "sample", "tutorial"]);
|
|
814
|
+
const SECURITY_TRACE_INTENT_TOKENS = new Set([
|
|
815
|
+
"auth",
|
|
816
|
+
"authorization",
|
|
817
|
+
"binding",
|
|
818
|
+
"config",
|
|
819
|
+
"enforce",
|
|
820
|
+
"mcp",
|
|
821
|
+
"project_root_path",
|
|
822
|
+
"security",
|
|
823
|
+
"session",
|
|
824
|
+
"stdio",
|
|
825
|
+
"tenant",
|
|
826
|
+
"token",
|
|
827
|
+
"workspace",
|
|
828
|
+
"workspace_id"
|
|
829
|
+
]);
|
|
486
830
|
const SOURCE_PATH_PREFIXES = ["src/", "app/", "apps/", "crates/", "internal/", "lib/", "package/", "packages/"];
|
|
487
831
|
const LOW_PRIORITY_PATH_PREFIXES = [
|
|
488
832
|
".next/",
|
|
@@ -543,6 +887,20 @@ function hasTestIntent(tokens) {
|
|
|
543
887
|
function hasExampleIntent(tokens) {
|
|
544
888
|
return tokens.some((token) => EXAMPLE_INTENT_TOKENS.has(token));
|
|
545
889
|
}
|
|
890
|
+
function hasSecurityTraceIntent(tokens, queryText) {
|
|
891
|
+
if (tokens.some((token) => SECURITY_TRACE_INTENT_TOKENS.has(token))) {
|
|
892
|
+
return true;
|
|
893
|
+
}
|
|
894
|
+
return /\btenant_id\b|\bworkspace_id\b|\bproject_root_path\b|\bRCE_[A-Z0-9_]{4,}\b/.test(queryText);
|
|
895
|
+
}
|
|
896
|
+
function isGuidanceOrMetaPath(path) {
|
|
897
|
+
const normalized = path.toLowerCase();
|
|
898
|
+
return (normalized.endsWith("mcp-tool-guidance.ts") ||
|
|
899
|
+
normalized.includes("/guidance/") ||
|
|
900
|
+
normalized.includes("/meta/") ||
|
|
901
|
+
normalized.includes("/_meta/") ||
|
|
902
|
+
normalized.includes("tool-guidance"));
|
|
903
|
+
}
|
|
546
904
|
function pathQualityBias(path, queryTokens, config, queryText) {
|
|
547
905
|
const normalizedPath = path.toLowerCase();
|
|
548
906
|
const docIntent = hasDocIntent(queryTokens);
|
|
@@ -553,6 +911,7 @@ function pathQualityBias(path, queryTokens, config, queryText) {
|
|
|
553
911
|
const uiComponentIntent = hasUiComponentIntent(queryTokens);
|
|
554
912
|
const testIntent = hasTestIntent(queryTokens);
|
|
555
913
|
const exampleIntent = hasExampleIntent(queryTokens);
|
|
914
|
+
const securityTraceIntent = hasSecurityTraceIntent(queryTokens, queryText ?? queryTokens.join(" "));
|
|
556
915
|
let bias = 0;
|
|
557
916
|
const pathBias = config.path_bias;
|
|
558
917
|
const isSourcePath = SOURCE_PATH_PREFIXES.some((prefix) => normalizedPath.startsWith(prefix) || normalizedPath.includes(`/${prefix}`));
|
|
@@ -616,6 +975,9 @@ function pathQualityBias(path, queryTokens, config, queryText) {
|
|
|
616
975
|
if (docsPreferred && isSourcePath) {
|
|
617
976
|
bias -= pathBias.doc_intent_source_penalty;
|
|
618
977
|
}
|
|
978
|
+
if (securityTraceIntent && !docsPreferred && isGuidanceOrMetaPath(normalizedPath)) {
|
|
979
|
+
bias -= pathBias.security_trace_meta_penalty;
|
|
980
|
+
}
|
|
619
981
|
if (workspaceManifestIntent && normalizedPath === "cargo.toml") {
|
|
620
982
|
bias += pathBias.workspace_manifest_root_boost;
|
|
621
983
|
}
|
|
@@ -681,13 +1043,19 @@ function buildChunks(file, chunkingConfig) {
|
|
|
681
1043
|
config: {
|
|
682
1044
|
strategy: chunkingConfig.strategy,
|
|
683
1045
|
fallback_strategy: chunkingConfig.fallback_strategy,
|
|
684
|
-
target_chunk_tokens:
|
|
685
|
-
chunk_overlap_tokens:
|
|
1046
|
+
target_chunk_tokens: chunkingConfig.target_chunk_tokens,
|
|
1047
|
+
chunk_overlap_tokens: chunkingConfig.chunk_overlap_tokens,
|
|
1048
|
+
budget_tokenizer: chunkingConfig.budget_tokenizer,
|
|
1049
|
+
boundary_strictness: chunkingConfig.boundary_strictness,
|
|
686
1050
|
max_chunks_per_file: MAX_CHUNKS_PER_FILE,
|
|
687
1051
|
parse_timeout_ms: chunkingConfig.parse_timeout_ms,
|
|
688
|
-
enabled_languages: chunkingConfig.enabled_languages
|
|
1052
|
+
enabled_languages: chunkingConfig.enabled_languages,
|
|
1053
|
+
recursive_semantic_chunking_enabled: chunkingConfig.recursive_semantic_chunking_enabled,
|
|
1054
|
+
semantic_merge_gap_lines: chunkingConfig.semantic_merge_gap_lines,
|
|
1055
|
+
semantic_merge_max_span_lines: chunkingConfig.semantic_merge_max_span_lines,
|
|
1056
|
+
comment_forward_absorb_enabled: chunkingConfig.comment_forward_absorb_enabled
|
|
689
1057
|
},
|
|
690
|
-
tokenize
|
|
1058
|
+
tokenize: (text) => chunkBudgetTokenize(text, chunkingConfig.budget_tokenizer)
|
|
691
1059
|
});
|
|
692
1060
|
return {
|
|
693
1061
|
chunks: chunkingResult.chunks.map((chunk) => ({
|
|
@@ -705,9 +1073,27 @@ function buildChunks(file, chunkingConfig) {
|
|
|
705
1073
|
parse_latency_ms: chunkingResult.parse_latency_ms,
|
|
706
1074
|
language_aware_attempt_latency_ms: chunkingResult.language_aware_attempt_latency_ms,
|
|
707
1075
|
fallback_path_latency_ms: chunkingResult.fallback_path_latency_ms,
|
|
708
|
-
language: chunkingResult.language
|
|
1076
|
+
language: chunkingResult.language,
|
|
1077
|
+
recursive_semantic_chunking_used: chunkingResult.recursive_semantic_chunking_used
|
|
709
1078
|
};
|
|
710
1079
|
}
|
|
1080
|
+
function buildChunkEmbeddingText(chunk, config, embeddingProviderId) {
|
|
1081
|
+
const isDeterministicProvider = embeddingProviderId.trim().toLowerCase() === "deterministic";
|
|
1082
|
+
if (!config.embedding_context_prefix_enabled || isDeterministicProvider) {
|
|
1083
|
+
return chunk.snippet;
|
|
1084
|
+
}
|
|
1085
|
+
const normalizedPath = normalizePath(chunk.path);
|
|
1086
|
+
const pathParts = normalizedPath.split("/").filter(Boolean);
|
|
1087
|
+
const contextPath = pathParts.length > 2 ? pathParts.slice(-2).join("/") : normalizedPath;
|
|
1088
|
+
const symbol = detectSnippetSymbolName(chunk.snippet);
|
|
1089
|
+
const linesLabel = `${chunk.start_line}-${chunk.end_line}`;
|
|
1090
|
+
const symbolLabel = symbol ? ` > ${symbol}` : "";
|
|
1091
|
+
const prefix = `${contextPath}:${linesLabel}${symbolLabel}`;
|
|
1092
|
+
return `${prefix}\n${chunk.snippet}`;
|
|
1093
|
+
}
|
|
1094
|
+
function buildChunkEmbeddingTexts(chunks, config, embeddingProviderId) {
|
|
1095
|
+
return chunks.map((chunk) => buildChunkEmbeddingText(chunk, config, embeddingProviderId));
|
|
1096
|
+
}
|
|
711
1097
|
function pseudoEmbedding(input, dimensions = 24) {
|
|
712
1098
|
const safeDimensions = Math.max(1, dimensions);
|
|
713
1099
|
let source = sha256(input);
|
|
@@ -732,6 +1118,138 @@ function sleep(ms) {
|
|
|
732
1118
|
setTimeout(resolve, ms);
|
|
733
1119
|
});
|
|
734
1120
|
}
|
|
1121
|
+
export class ProviderRateLimitExceededError extends Error {
|
|
1122
|
+
retry_after_ms;
|
|
1123
|
+
constructor(message, retry_after_ms) {
|
|
1124
|
+
super(message);
|
|
1125
|
+
this.retry_after_ms = retry_after_ms;
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
export class LocalProviderRequestLimiter {
|
|
1129
|
+
mode = "local";
|
|
1130
|
+
buckets = new Map();
|
|
1131
|
+
now;
|
|
1132
|
+
sleeper;
|
|
1133
|
+
constructor(options) {
|
|
1134
|
+
this.now = options?.now ?? (() => Date.now());
|
|
1135
|
+
this.sleeper = options?.sleeper ?? sleep;
|
|
1136
|
+
}
|
|
1137
|
+
async acquire(input) {
|
|
1138
|
+
if (!Number.isInteger(input.max_requests_per_minute) || input.max_requests_per_minute <= 0) {
|
|
1139
|
+
throw new Error("provider limiter requires max_requests_per_minute to be a positive integer");
|
|
1140
|
+
}
|
|
1141
|
+
if (!Number.isInteger(input.max_wait_ms) || input.max_wait_ms < 0) {
|
|
1142
|
+
throw new Error("provider limiter requires max_wait_ms to be a non-negative integer");
|
|
1143
|
+
}
|
|
1144
|
+
const refillPerMs = input.max_requests_per_minute / 60_000;
|
|
1145
|
+
let waitedMs = 0;
|
|
1146
|
+
const deadline = this.now() + input.max_wait_ms;
|
|
1147
|
+
while (true) {
|
|
1148
|
+
const nowMs = this.now();
|
|
1149
|
+
let bucket = this.buckets.get(input.scope);
|
|
1150
|
+
if (!bucket) {
|
|
1151
|
+
bucket = {
|
|
1152
|
+
tokens: input.max_requests_per_minute,
|
|
1153
|
+
last_refill_ms: nowMs
|
|
1154
|
+
};
|
|
1155
|
+
this.buckets.set(input.scope, bucket);
|
|
1156
|
+
}
|
|
1157
|
+
if (nowMs > bucket.last_refill_ms) {
|
|
1158
|
+
const elapsedMs = nowMs - bucket.last_refill_ms;
|
|
1159
|
+
bucket.tokens = Math.min(input.max_requests_per_minute, bucket.tokens + elapsedMs * refillPerMs);
|
|
1160
|
+
bucket.last_refill_ms = nowMs;
|
|
1161
|
+
}
|
|
1162
|
+
if (bucket.tokens >= 1) {
|
|
1163
|
+
bucket.tokens -= 1;
|
|
1164
|
+
return { wait_ms: waitedMs };
|
|
1165
|
+
}
|
|
1166
|
+
const retryAfterMs = Math.max(1, Math.ceil((1 - bucket.tokens) / refillPerMs));
|
|
1167
|
+
const remainingMs = deadline - nowMs;
|
|
1168
|
+
if (remainingMs <= 0 || retryAfterMs > remainingMs) {
|
|
1169
|
+
throw new ProviderRateLimitExceededError(`provider request rate limit exceeded for scope "${input.scope}"`, Math.max(1, retryAfterMs));
|
|
1170
|
+
}
|
|
1171
|
+
const sleepMs = Math.max(1, Math.min(retryAfterMs, remainingMs));
|
|
1172
|
+
await this.sleeper(sleepMs);
|
|
1173
|
+
waitedMs += sleepMs;
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
const REDIS_PROVIDER_LIMITER_SCRIPT = `
|
|
1178
|
+
local key = KEYS[1]
|
|
1179
|
+
local limit = tonumber(ARGV[1])
|
|
1180
|
+
local window_ms = tonumber(ARGV[2])
|
|
1181
|
+
local count = redis.call("INCR", key)
|
|
1182
|
+
if count == 1 then
|
|
1183
|
+
redis.call("PEXPIRE", key, window_ms)
|
|
1184
|
+
end
|
|
1185
|
+
if count <= limit then
|
|
1186
|
+
return {1, 0}
|
|
1187
|
+
end
|
|
1188
|
+
local ttl = redis.call("PTTL", key)
|
|
1189
|
+
if ttl < 0 then
|
|
1190
|
+
ttl = window_ms
|
|
1191
|
+
end
|
|
1192
|
+
return {0, ttl}
|
|
1193
|
+
`;
|
|
1194
|
+
export class RedisProviderRequestLimiter {
|
|
1195
|
+
mode = "redis";
|
|
1196
|
+
redis;
|
|
1197
|
+
keyPrefix;
|
|
1198
|
+
windowMs;
|
|
1199
|
+
now;
|
|
1200
|
+
sleeper;
|
|
1201
|
+
constructor(options) {
|
|
1202
|
+
if (!options.redis || typeof options.redis.eval !== "function") {
|
|
1203
|
+
throw new Error("invalid redis provider limiter config: redis client with eval() is required");
|
|
1204
|
+
}
|
|
1205
|
+
this.redis = options.redis;
|
|
1206
|
+
this.keyPrefix = options.key_prefix?.trim() || "rce:provider_rate_limit";
|
|
1207
|
+
this.windowMs = options.window_ms ?? 60_000;
|
|
1208
|
+
this.now = options.now ?? (() => Date.now());
|
|
1209
|
+
this.sleeper = options.sleeper ?? sleep;
|
|
1210
|
+
if (!Number.isInteger(this.windowMs) || this.windowMs <= 0) {
|
|
1211
|
+
throw new Error("invalid redis provider limiter config: window_ms must be a positive integer");
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
async acquire(input) {
|
|
1215
|
+
if (!Number.isInteger(input.max_requests_per_minute) || input.max_requests_per_minute <= 0) {
|
|
1216
|
+
throw new Error("provider limiter requires max_requests_per_minute to be a positive integer");
|
|
1217
|
+
}
|
|
1218
|
+
if (!Number.isInteger(input.max_wait_ms) || input.max_wait_ms < 0) {
|
|
1219
|
+
throw new Error("provider limiter requires max_wait_ms to be a non-negative integer");
|
|
1220
|
+
}
|
|
1221
|
+
let waitedMs = 0;
|
|
1222
|
+
const deadline = this.now() + input.max_wait_ms;
|
|
1223
|
+
while (true) {
|
|
1224
|
+
const attempt = await this.reserveAttempt(input.scope, input.max_requests_per_minute);
|
|
1225
|
+
if (attempt.allowed) {
|
|
1226
|
+
return { wait_ms: waitedMs };
|
|
1227
|
+
}
|
|
1228
|
+
const nowMs = this.now();
|
|
1229
|
+
const remainingMs = deadline - nowMs;
|
|
1230
|
+
const retryAfterMs = Math.max(1, attempt.retry_after_ms);
|
|
1231
|
+
if (remainingMs <= 0 || retryAfterMs > remainingMs) {
|
|
1232
|
+
throw new ProviderRateLimitExceededError(`provider request rate limit exceeded for scope "${input.scope}"`, retryAfterMs);
|
|
1233
|
+
}
|
|
1234
|
+
const sleepMs = Math.max(1, Math.min(retryAfterMs, remainingMs));
|
|
1235
|
+
await this.sleeper(sleepMs);
|
|
1236
|
+
waitedMs += sleepMs;
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
async reserveAttempt(scope, maxRequestsPerMinute) {
|
|
1240
|
+
const key = `${this.keyPrefix}:${scope}`;
|
|
1241
|
+
const raw = await this.redis.eval(REDIS_PROVIDER_LIMITER_SCRIPT, 1, key, maxRequestsPerMinute, this.windowMs);
|
|
1242
|
+
if (Array.isArray(raw)) {
|
|
1243
|
+
const allowed = Number(raw[0] ?? 0) === 1;
|
|
1244
|
+
const retryAfterMs = Number(raw[1] ?? 0);
|
|
1245
|
+
return {
|
|
1246
|
+
allowed,
|
|
1247
|
+
retry_after_ms: Number.isFinite(retryAfterMs) ? Math.max(0, Math.trunc(retryAfterMs)) : this.windowMs
|
|
1248
|
+
};
|
|
1249
|
+
}
|
|
1250
|
+
throw new Error("redis provider limiter returned unexpected eval() payload");
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
735
1253
|
export class DeterministicEmbeddingProvider {
|
|
736
1254
|
dimensions;
|
|
737
1255
|
model;
|
|
@@ -762,6 +1280,12 @@ export class OpenAICompatibleEmbeddingProvider {
|
|
|
762
1280
|
timeoutMs;
|
|
763
1281
|
batchSize;
|
|
764
1282
|
maxRetries;
|
|
1283
|
+
transientForbiddenMaxRetries;
|
|
1284
|
+
requestLimiter;
|
|
1285
|
+
requestLimitScope;
|
|
1286
|
+
maxRequestsPerMinute;
|
|
1287
|
+
indexMaxWaitMs;
|
|
1288
|
+
queryMaxWaitMs;
|
|
765
1289
|
observability;
|
|
766
1290
|
constructor(options) {
|
|
767
1291
|
const baseUrl = options.base_url.trim().replace(/\/+$/, "");
|
|
@@ -780,6 +1304,17 @@ export class OpenAICompatibleEmbeddingProvider {
|
|
|
780
1304
|
this.timeoutMs = options.timeout_ms ?? DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_TIMEOUT_MS;
|
|
781
1305
|
this.batchSize = options.batch_size ?? DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_BATCH_SIZE;
|
|
782
1306
|
this.maxRetries = options.max_retries ?? DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_MAX_RETRIES;
|
|
1307
|
+
this.transientForbiddenMaxRetries =
|
|
1308
|
+
options.transient_forbidden_max_retries ?? DEFAULT_OPENAI_COMPATIBLE_EMBEDDING_TRANSIENT_403_MAX_RETRIES;
|
|
1309
|
+
this.requestLimiter = options.request_limiter;
|
|
1310
|
+
this.requestLimitScope = resolveProviderLimiterScope({
|
|
1311
|
+
provider: "openai_compatible",
|
|
1312
|
+
apiKey,
|
|
1313
|
+
overrideScopeId: options.request_limit_scope_id
|
|
1314
|
+
});
|
|
1315
|
+
this.maxRequestsPerMinute = options.max_requests_per_minute ?? DEFAULT_PROVIDER_MAX_REQUESTS_PER_MINUTE;
|
|
1316
|
+
this.indexMaxWaitMs = options.index_max_wait_ms ?? DEFAULT_PROVIDER_LIMIT_INDEX_MAX_WAIT_MS;
|
|
1317
|
+
this.queryMaxWaitMs = options.query_max_wait_ms ?? DEFAULT_PROVIDER_LIMIT_QUERY_MAX_WAIT_MS;
|
|
783
1318
|
this.observability = options.observability ?? getObservability("retrieval-core");
|
|
784
1319
|
if (!Number.isInteger(this.dimensions) || this.dimensions <= 0) {
|
|
785
1320
|
throw new Error("invalid openai-compatible embedding config: dimensions must be a positive integer");
|
|
@@ -793,6 +1328,18 @@ export class OpenAICompatibleEmbeddingProvider {
|
|
|
793
1328
|
if (!Number.isInteger(this.maxRetries) || this.maxRetries < 0) {
|
|
794
1329
|
throw new Error("invalid openai-compatible embedding config: max_retries must be a non-negative integer");
|
|
795
1330
|
}
|
|
1331
|
+
if (!Number.isInteger(this.transientForbiddenMaxRetries) || this.transientForbiddenMaxRetries < 0) {
|
|
1332
|
+
throw new Error("invalid openai-compatible embedding config: transient_forbidden_max_retries must be a non-negative integer");
|
|
1333
|
+
}
|
|
1334
|
+
if (!Number.isInteger(this.maxRequestsPerMinute) || this.maxRequestsPerMinute <= 0) {
|
|
1335
|
+
throw new Error("invalid openai-compatible embedding config: max_requests_per_minute must be a positive integer");
|
|
1336
|
+
}
|
|
1337
|
+
if (!Number.isInteger(this.indexMaxWaitMs) || this.indexMaxWaitMs < 0) {
|
|
1338
|
+
throw new Error("invalid openai-compatible embedding config: index_max_wait_ms must be a non-negative integer");
|
|
1339
|
+
}
|
|
1340
|
+
if (!Number.isInteger(this.queryMaxWaitMs) || this.queryMaxWaitMs < 0) {
|
|
1341
|
+
throw new Error("invalid openai-compatible embedding config: query_max_wait_ms must be a non-negative integer");
|
|
1342
|
+
}
|
|
796
1343
|
}
|
|
797
1344
|
describe() {
|
|
798
1345
|
return {
|
|
@@ -819,11 +1366,12 @@ export class OpenAICompatibleEmbeddingProvider {
|
|
|
819
1366
|
model: this.model,
|
|
820
1367
|
purpose
|
|
821
1368
|
};
|
|
822
|
-
|
|
1369
|
+
let attempt = 0;
|
|
1370
|
+
while (true) {
|
|
823
1371
|
const startedAt = Date.now();
|
|
824
1372
|
this.observability.metrics.increment("retrieval_embedding_provider_requests_total", 1, labels);
|
|
825
1373
|
try {
|
|
826
|
-
return await this.embedBatchOnce(texts);
|
|
1374
|
+
return await this.embedBatchOnce(texts, purpose);
|
|
827
1375
|
}
|
|
828
1376
|
catch (error) {
|
|
829
1377
|
const failure = this.toProviderFailure(error);
|
|
@@ -831,30 +1379,69 @@ export class OpenAICompatibleEmbeddingProvider {
|
|
|
831
1379
|
...labels,
|
|
832
1380
|
reason: failure.reason
|
|
833
1381
|
});
|
|
834
|
-
const
|
|
1382
|
+
const maxRetriesForFailure = this.maxRetriesForReason(failure.reason);
|
|
1383
|
+
const shouldRetry = failure.retryable && attempt < maxRetriesForFailure;
|
|
835
1384
|
this.observability.logger.warn("embedding provider request failed", {
|
|
836
1385
|
provider: "openai_compatible",
|
|
837
1386
|
model: this.model,
|
|
838
1387
|
purpose,
|
|
839
1388
|
reason: failure.reason,
|
|
1389
|
+
provider_message: failure.message,
|
|
840
1390
|
retryable: failure.retryable,
|
|
841
1391
|
retrying: shouldRetry,
|
|
842
1392
|
attempt: attempt + 1,
|
|
843
|
-
max_attempts:
|
|
1393
|
+
max_attempts: maxRetriesForFailure + 1,
|
|
1394
|
+
retry_after_ms: failure.retry_after_ms
|
|
844
1395
|
});
|
|
845
1396
|
if (shouldRetry) {
|
|
846
|
-
await sleep(this.retryDelayMs(attempt));
|
|
1397
|
+
await sleep(this.retryDelayMs(attempt, failure));
|
|
1398
|
+
attempt += 1;
|
|
847
1399
|
continue;
|
|
848
1400
|
}
|
|
1401
|
+
if (failure.reason === "client_rate_limited" || failure.reason === "rate_limited") {
|
|
1402
|
+
throw new RetrievalError("RATE_LIMITED", `embedding provider rate limited; ${failure.message}`);
|
|
1403
|
+
}
|
|
849
1404
|
throw new RetrievalError("UPSTREAM_FAILURE", `embedding provider request failed (${failure.reason}); ${failure.message}`);
|
|
850
1405
|
}
|
|
851
1406
|
finally {
|
|
852
1407
|
this.observability.metrics.observe("retrieval_embedding_provider_latency_ms", Date.now() - startedAt, labels);
|
|
853
1408
|
}
|
|
854
1409
|
}
|
|
855
|
-
throw new RetrievalError("UPSTREAM_FAILURE", "embedding provider retries exhausted");
|
|
856
1410
|
}
|
|
857
|
-
async
|
|
1411
|
+
async enforceRequestLimit(purpose) {
|
|
1412
|
+
if (!this.requestLimiter) {
|
|
1413
|
+
return;
|
|
1414
|
+
}
|
|
1415
|
+
const maxWaitMs = purpose === "index" ? this.indexMaxWaitMs : this.queryMaxWaitMs;
|
|
1416
|
+
const labels = {
|
|
1417
|
+
provider: "openai_compatible",
|
|
1418
|
+
model: this.model,
|
|
1419
|
+
purpose,
|
|
1420
|
+
limiter_mode: this.requestLimiter.mode ?? "custom"
|
|
1421
|
+
};
|
|
1422
|
+
try {
|
|
1423
|
+
const acquired = await this.requestLimiter.acquire({
|
|
1424
|
+
scope: this.requestLimitScope,
|
|
1425
|
+
max_requests_per_minute: this.maxRequestsPerMinute,
|
|
1426
|
+
max_wait_ms: maxWaitMs
|
|
1427
|
+
});
|
|
1428
|
+
this.observability.metrics.observe("retrieval_provider_limiter_wait_ms", acquired.wait_ms, labels);
|
|
1429
|
+
this.observability.metrics.increment("retrieval_provider_requests_shaped_total", 1, labels);
|
|
1430
|
+
}
|
|
1431
|
+
catch (error) {
|
|
1432
|
+
this.observability.metrics.increment("retrieval_provider_limiter_block_total", 1, {
|
|
1433
|
+
...labels,
|
|
1434
|
+
reason: "wait_timeout"
|
|
1435
|
+
});
|
|
1436
|
+
if (error instanceof ProviderRateLimitExceededError) {
|
|
1437
|
+
const retryable = purpose === "index";
|
|
1438
|
+
throw new EmbeddingProviderRequestError("client_rate_limited", retryable, `${error.message}; retry_after_ms=${error.retry_after_ms}`, error.retry_after_ms);
|
|
1439
|
+
}
|
|
1440
|
+
throw error;
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
async embedBatchOnce(texts, purpose) {
|
|
1444
|
+
await this.enforceRequestLimit(purpose);
|
|
858
1445
|
const controller = new AbortController();
|
|
859
1446
|
const timeoutId = setTimeout(() => {
|
|
860
1447
|
controller.abort();
|
|
@@ -887,13 +1474,18 @@ export class OpenAICompatibleEmbeddingProvider {
|
|
|
887
1474
|
if (!response.ok) {
|
|
888
1475
|
const details = await safeResponseText(response);
|
|
889
1476
|
if (response.status === 429) {
|
|
890
|
-
throw new EmbeddingProviderRequestError("rate_limited", true, `HTTP 429 ${details}`.trim());
|
|
1477
|
+
throw new EmbeddingProviderRequestError("rate_limited", true, `HTTP 429 ${details}`.trim(), parseRetryAfterMs(response.headers.get("retry-after")));
|
|
891
1478
|
}
|
|
892
1479
|
if (response.status >= 500) {
|
|
893
1480
|
throw new EmbeddingProviderRequestError("http_5xx", true, `HTTP ${response.status} ${details}`.trim());
|
|
894
1481
|
}
|
|
895
|
-
if (response.status === 401
|
|
896
|
-
throw new EmbeddingProviderRequestError("auth_error", false, `HTTP
|
|
1482
|
+
if (response.status === 401) {
|
|
1483
|
+
throw new EmbeddingProviderRequestError("auth_error", false, `HTTP 401 ${details}`.trim());
|
|
1484
|
+
}
|
|
1485
|
+
if (response.status === 403) {
|
|
1486
|
+
const retryAfterMs = parseRetryAfterMs(response.headers.get("retry-after"));
|
|
1487
|
+
const retryable = this.isTransientForbidden(details, retryAfterMs);
|
|
1488
|
+
throw new EmbeddingProviderRequestError(retryable ? "forbidden_transient" : "auth_error", retryable, `HTTP 403 ${details}`.trim(), retryAfterMs);
|
|
897
1489
|
}
|
|
898
1490
|
if (response.status === 404) {
|
|
899
1491
|
throw new EmbeddingProviderRequestError("endpoint_not_found", false, `HTTP 404 ${details}`.trim());
|
|
@@ -945,16 +1537,81 @@ export class OpenAICompatibleEmbeddingProvider {
|
|
|
945
1537
|
}
|
|
946
1538
|
return vectors;
|
|
947
1539
|
}
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
1540
|
+
maxRetriesForReason(reason) {
|
|
1541
|
+
if (reason === "forbidden_transient") {
|
|
1542
|
+
return Math.max(this.maxRetries, this.transientForbiddenMaxRetries);
|
|
1543
|
+
}
|
|
1544
|
+
return this.maxRetries;
|
|
1545
|
+
}
|
|
1546
|
+
retryDelayMs(attempt, failure) {
|
|
1547
|
+
const baseBackoffMs = failure.reason === "forbidden_transient"
|
|
1548
|
+
? Math.min(2_500, 250 * 2 ** attempt)
|
|
1549
|
+
: 100 * (attempt + 1);
|
|
1550
|
+
const jitterMs = failure.reason === "forbidden_transient" ? Math.floor(Math.random() * 150) : Math.floor(Math.random() * 75);
|
|
1551
|
+
const computedDelayMs = baseBackoffMs + jitterMs;
|
|
1552
|
+
if (failure.retry_after_ms === undefined) {
|
|
1553
|
+
return computedDelayMs;
|
|
1554
|
+
}
|
|
1555
|
+
return Math.max(computedDelayMs, Math.max(1, failure.retry_after_ms));
|
|
1556
|
+
}
|
|
1557
|
+
isTransientForbidden(details, retryAfterMs) {
|
|
1558
|
+
if (retryAfterMs !== undefined) {
|
|
1559
|
+
return true;
|
|
1560
|
+
}
|
|
1561
|
+
const normalized = details.trim().toLowerCase();
|
|
1562
|
+
if (normalized.length === 0) {
|
|
1563
|
+
return false;
|
|
1564
|
+
}
|
|
1565
|
+
const transientSignals = [
|
|
1566
|
+
"rate limit",
|
|
1567
|
+
"too many requests",
|
|
1568
|
+
"temporar",
|
|
1569
|
+
"try again",
|
|
1570
|
+
"upstream",
|
|
1571
|
+
"timeout",
|
|
1572
|
+
"busy",
|
|
1573
|
+
"capacity",
|
|
1574
|
+
"bad_response_status_code"
|
|
1575
|
+
];
|
|
1576
|
+
if (transientSignals.some((signal) => normalized.includes(signal))) {
|
|
1577
|
+
return true;
|
|
1578
|
+
}
|
|
1579
|
+
const hardFailureSignals = [
|
|
1580
|
+
"invalid api key",
|
|
1581
|
+
"incorrect api key",
|
|
1582
|
+
"authentication",
|
|
1583
|
+
"unauthorized",
|
|
1584
|
+
"insufficient permissions",
|
|
1585
|
+
"insufficient scope",
|
|
1586
|
+
"permission denied",
|
|
1587
|
+
"organization not found",
|
|
1588
|
+
"account disabled",
|
|
1589
|
+
"insufficient quota",
|
|
1590
|
+
"quota exceeded",
|
|
1591
|
+
"billing",
|
|
1592
|
+
"credit",
|
|
1593
|
+
"payment required",
|
|
1594
|
+
"model not found",
|
|
1595
|
+
"unknown model",
|
|
1596
|
+
"unsupported model",
|
|
1597
|
+
"not allowed"
|
|
1598
|
+
];
|
|
1599
|
+
if (hardFailureSignals.some((signal) => normalized.includes(signal))) {
|
|
1600
|
+
return false;
|
|
1601
|
+
}
|
|
1602
|
+
return false;
|
|
952
1603
|
}
|
|
953
1604
|
toProviderFailure(error) {
|
|
954
1605
|
if (error instanceof EmbeddingProviderRequestError) {
|
|
955
1606
|
return error;
|
|
956
1607
|
}
|
|
1608
|
+
if (error instanceof ProviderRateLimitExceededError) {
|
|
1609
|
+
return new EmbeddingProviderRequestError("client_rate_limited", false, `${error.message}; retry_after_ms=${error.retry_after_ms}`);
|
|
1610
|
+
}
|
|
957
1611
|
if (error instanceof RetrievalError) {
|
|
1612
|
+
if (error.code === "RATE_LIMITED") {
|
|
1613
|
+
return new EmbeddingProviderRequestError("client_rate_limited", false, error.message);
|
|
1614
|
+
}
|
|
958
1615
|
return new EmbeddingProviderRequestError("upstream_failure", false, error.message);
|
|
959
1616
|
}
|
|
960
1617
|
if (error instanceof Error) {
|
|
@@ -963,45 +1620,910 @@ export class OpenAICompatibleEmbeddingProvider {
|
|
|
963
1620
|
return new EmbeddingProviderRequestError("unknown_error", false, String(error));
|
|
964
1621
|
}
|
|
965
1622
|
}
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
1623
|
+
export class OpenAICompatibleRerankerProvider {
|
|
1624
|
+
endpoint;
|
|
1625
|
+
apiKey;
|
|
1626
|
+
model;
|
|
1627
|
+
timeoutMs;
|
|
1628
|
+
requestLimiter;
|
|
1629
|
+
requestLimitScope;
|
|
1630
|
+
maxRequestsPerMinute;
|
|
1631
|
+
rerankMaxWaitMs;
|
|
1632
|
+
observability;
|
|
1633
|
+
constructor(options) {
|
|
1634
|
+
const baseUrl = options.base_url.trim().replace(/\/+$/, "");
|
|
1635
|
+
if (baseUrl.length === 0) {
|
|
1636
|
+
throw new Error("invalid openai-compatible reranker config: base_url must be non-empty");
|
|
1637
|
+
}
|
|
1638
|
+
const apiKey = options.api_key.trim();
|
|
1639
|
+
if (apiKey.length === 0) {
|
|
1640
|
+
throw new Error("invalid openai-compatible reranker config: api_key must be non-empty");
|
|
1641
|
+
}
|
|
1642
|
+
this.endpoint = `${baseUrl}/rerank`;
|
|
1643
|
+
this.apiKey = apiKey;
|
|
1644
|
+
this.model = options.model?.trim() || DEFAULT_OPENAI_COMPATIBLE_RERANKER_MODEL;
|
|
1645
|
+
this.timeoutMs = options.timeout_ms ?? DEFAULT_OPENAI_COMPATIBLE_RERANKER_TIMEOUT_MS;
|
|
1646
|
+
this.requestLimiter = options.request_limiter;
|
|
1647
|
+
this.requestLimitScope = resolveProviderLimiterScope({
|
|
1648
|
+
provider: "openai_compatible",
|
|
1649
|
+
apiKey,
|
|
1650
|
+
overrideScopeId: options.request_limit_scope_id
|
|
1651
|
+
});
|
|
1652
|
+
this.maxRequestsPerMinute = options.max_requests_per_minute ?? DEFAULT_PROVIDER_MAX_REQUESTS_PER_MINUTE;
|
|
1653
|
+
this.rerankMaxWaitMs = options.rerank_max_wait_ms ?? DEFAULT_PROVIDER_LIMIT_RERANK_MAX_WAIT_MS;
|
|
1654
|
+
this.observability = options.observability ?? getObservability("retrieval-core");
|
|
1655
|
+
if (!Number.isInteger(this.timeoutMs) || this.timeoutMs <= 0) {
|
|
1656
|
+
throw new Error("invalid openai-compatible reranker config: timeout_ms must be a positive integer");
|
|
1657
|
+
}
|
|
1658
|
+
if (!Number.isInteger(this.maxRequestsPerMinute) || this.maxRequestsPerMinute <= 0) {
|
|
1659
|
+
throw new Error("invalid openai-compatible reranker config: max_requests_per_minute must be a positive integer");
|
|
1660
|
+
}
|
|
1661
|
+
if (!Number.isInteger(this.rerankMaxWaitMs) || this.rerankMaxWaitMs < 0) {
|
|
1662
|
+
throw new Error("invalid openai-compatible reranker config: rerank_max_wait_ms must be a non-negative integer");
|
|
1663
|
+
}
|
|
973
1664
|
}
|
|
974
|
-
|
|
975
|
-
function resolveEmbeddingDescriptor(provider) {
|
|
976
|
-
const described = provider.describe?.();
|
|
977
|
-
if (!described) {
|
|
1665
|
+
describe() {
|
|
978
1666
|
return {
|
|
979
|
-
provider: "
|
|
980
|
-
|
|
1667
|
+
provider: "openai_compatible",
|
|
1668
|
+
model: this.model
|
|
981
1669
|
};
|
|
982
1670
|
}
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1671
|
+
async rerank(input) {
|
|
1672
|
+
if (input.documents.length === 0) {
|
|
1673
|
+
return [];
|
|
1674
|
+
}
|
|
1675
|
+
await this.enforceRequestLimit();
|
|
1676
|
+
const topN = Math.max(1, Math.min(input.top_n, input.documents.length));
|
|
1677
|
+
const controller = new AbortController();
|
|
1678
|
+
const timeoutId = setTimeout(() => {
|
|
1679
|
+
controller.abort();
|
|
1680
|
+
}, this.timeoutMs);
|
|
1681
|
+
let response;
|
|
1682
|
+
try {
|
|
1683
|
+
response = await fetch(this.endpoint, {
|
|
1684
|
+
method: "POST",
|
|
1685
|
+
headers: {
|
|
1686
|
+
authorization: `Bearer ${this.apiKey}`,
|
|
1687
|
+
"content-type": "application/json"
|
|
1688
|
+
},
|
|
1689
|
+
body: JSON.stringify({
|
|
1690
|
+
model: this.model,
|
|
1691
|
+
query: input.query,
|
|
1692
|
+
documents: input.documents,
|
|
1693
|
+
top_n: topN
|
|
1694
|
+
}),
|
|
1695
|
+
signal: controller.signal
|
|
1696
|
+
});
|
|
1697
|
+
}
|
|
1698
|
+
catch (error) {
|
|
1699
|
+
if (error && typeof error === "object" && "name" in error && error.name === "AbortError") {
|
|
1700
|
+
throw new RerankerProviderRequestError("timeout", `request timed out after ${this.timeoutMs}ms`);
|
|
1701
|
+
}
|
|
1702
|
+
throw new RerankerProviderRequestError("network_error", error instanceof Error ? error.message : String(error));
|
|
1703
|
+
}
|
|
1704
|
+
finally {
|
|
1705
|
+
clearTimeout(timeoutId);
|
|
1706
|
+
}
|
|
1707
|
+
if (!response.ok) {
|
|
1708
|
+
const details = await safeResponseText(response);
|
|
1709
|
+
if (response.status === 429) {
|
|
1710
|
+
throw new RerankerProviderRequestError("rate_limited", `HTTP 429 ${details}`.trim());
|
|
1711
|
+
}
|
|
1712
|
+
if (response.status === 401 || response.status === 403) {
|
|
1713
|
+
throw new RerankerProviderRequestError("auth_error", `HTTP ${response.status} ${details}`.trim());
|
|
1714
|
+
}
|
|
1715
|
+
if (response.status === 404) {
|
|
1716
|
+
throw new RerankerProviderRequestError("endpoint_not_found", `HTTP 404 ${details}`.trim());
|
|
1717
|
+
}
|
|
1718
|
+
if (response.status >= 500) {
|
|
1719
|
+
throw new RerankerProviderRequestError("http_5xx", `HTTP ${response.status} ${details}`.trim());
|
|
1720
|
+
}
|
|
1721
|
+
throw new RerankerProviderRequestError("http_4xx", `HTTP ${response.status} ${details}`.trim());
|
|
1722
|
+
}
|
|
1723
|
+
let payload;
|
|
1724
|
+
try {
|
|
1725
|
+
payload = await response.json();
|
|
1726
|
+
}
|
|
1727
|
+
catch {
|
|
1728
|
+
throw new RerankerProviderRequestError("invalid_json", "provider returned non-JSON response");
|
|
1729
|
+
}
|
|
1730
|
+
if (!payload || typeof payload !== "object") {
|
|
1731
|
+
throw new RerankerProviderRequestError("invalid_response", "provider response must be an object");
|
|
1732
|
+
}
|
|
1733
|
+
const maybeResults = "results" in payload ? payload.results : payload.data;
|
|
1734
|
+
if (!Array.isArray(maybeResults)) {
|
|
1735
|
+
throw new RerankerProviderRequestError("invalid_response", "provider response missing results array");
|
|
1736
|
+
}
|
|
1737
|
+
const output = [];
|
|
1738
|
+
for (const row of maybeResults) {
|
|
1739
|
+
if (!row || typeof row !== "object") {
|
|
1740
|
+
throw new RerankerProviderRequestError("invalid_response", "rerank row must be an object");
|
|
1741
|
+
}
|
|
1742
|
+
const rawIndex = row.index;
|
|
1743
|
+
if (!Number.isInteger(rawIndex)) {
|
|
1744
|
+
throw new RerankerProviderRequestError("invalid_response", "rerank row index must be an integer");
|
|
1745
|
+
}
|
|
1746
|
+
const index = rawIndex;
|
|
1747
|
+
if (index < 0 || index >= input.documents.length) {
|
|
1748
|
+
throw new RerankerProviderRequestError("invalid_response", "rerank row index out of range");
|
|
1749
|
+
}
|
|
1750
|
+
const rawScore = row.relevance_score ?? row.score;
|
|
1751
|
+
if (typeof rawScore !== "number" || !Number.isFinite(rawScore)) {
|
|
1752
|
+
throw new RerankerProviderRequestError("invalid_response", "rerank row score must be finite");
|
|
1753
|
+
}
|
|
1754
|
+
output.push({
|
|
1755
|
+
index,
|
|
1756
|
+
relevance_score: rawScore
|
|
1757
|
+
});
|
|
1758
|
+
}
|
|
1759
|
+
const seen = new Set();
|
|
1760
|
+
const ordered = [...output]
|
|
1761
|
+
.sort((a, b) => b.relevance_score - a.relevance_score || a.index - b.index)
|
|
1762
|
+
.filter((row) => {
|
|
1763
|
+
if (seen.has(row.index)) {
|
|
1764
|
+
return false;
|
|
1765
|
+
}
|
|
1766
|
+
seen.add(row.index);
|
|
1767
|
+
return true;
|
|
1768
|
+
})
|
|
1769
|
+
.slice(0, topN);
|
|
1770
|
+
if (ordered.length === 0) {
|
|
1771
|
+
throw new RerankerProviderRequestError("invalid_response", "provider returned zero rerank results");
|
|
1772
|
+
}
|
|
1773
|
+
return ordered;
|
|
1774
|
+
}
|
|
1775
|
+
async enforceRequestLimit() {
|
|
1776
|
+
if (!this.requestLimiter) {
|
|
1777
|
+
return;
|
|
1778
|
+
}
|
|
1779
|
+
const labels = {
|
|
1780
|
+
provider: "openai_compatible",
|
|
1781
|
+
model: this.model,
|
|
1782
|
+
purpose: "rerank",
|
|
1783
|
+
limiter_mode: this.requestLimiter.mode ?? "custom"
|
|
1784
|
+
};
|
|
1785
|
+
try {
|
|
1786
|
+
const acquired = await this.requestLimiter.acquire({
|
|
1787
|
+
scope: this.requestLimitScope,
|
|
1788
|
+
max_requests_per_minute: this.maxRequestsPerMinute,
|
|
1789
|
+
max_wait_ms: this.rerankMaxWaitMs
|
|
1790
|
+
});
|
|
1791
|
+
this.observability.metrics.observe("retrieval_provider_limiter_wait_ms", acquired.wait_ms, labels);
|
|
1792
|
+
this.observability.metrics.increment("retrieval_provider_requests_shaped_total", 1, labels);
|
|
1793
|
+
}
|
|
1794
|
+
catch (error) {
|
|
1795
|
+
this.observability.metrics.increment("retrieval_provider_limiter_block_total", 1, {
|
|
1796
|
+
...labels,
|
|
1797
|
+
reason: "wait_timeout"
|
|
1798
|
+
});
|
|
1799
|
+
if (error instanceof ProviderRateLimitExceededError) {
|
|
1800
|
+
throw new RerankerProviderRequestError("rate_limited", `${error.message}; retry_after_ms=${error.retry_after_ms}`);
|
|
1801
|
+
}
|
|
1802
|
+
throw error;
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
function buildClaudeEnhancerSystemInstruction(language, style) {
|
|
1807
|
+
const languageRule = language === "zh"
|
|
1808
|
+
? "Output language must be Simplified Chinese."
|
|
1809
|
+
: language === "es"
|
|
1810
|
+
? "Output language must be Spanish."
|
|
1811
|
+
: "Output language must be English.";
|
|
1812
|
+
const styleRule = style === "lean"
|
|
1813
|
+
? "Style is lean: keep the response compact (roughly 90-180 words), avoid extra headings, and include only essential steps."
|
|
1814
|
+
: style === "deep"
|
|
1815
|
+
? "Style is deep: provide comprehensive but grounded guidance (roughly 260-420 words) with concrete constraints, edge cases, and validation."
|
|
1816
|
+
: "Style is standard: provide balanced depth (roughly 160-300 words) with clear scope, steps, and validation.";
|
|
1817
|
+
return [
|
|
1818
|
+
"You are a high-precision prompt enhancement agent for software engineering tasks.",
|
|
1819
|
+
languageRule,
|
|
1820
|
+
styleRule,
|
|
1821
|
+
"Return plain text only: the final enhanced prompt.",
|
|
1822
|
+
"Do not include markdown code fences.",
|
|
1823
|
+
"Preserve user intent exactly; do not add unrelated features.",
|
|
1824
|
+
"Do not invent file paths or symbols that are not present in provided context.",
|
|
1825
|
+
"Produce concise execution-ready prompts, not long generic templates.",
|
|
1826
|
+
"Prefer practical sections only: objective, scoped constraints, codebase anchors, implementation steps, validation.",
|
|
1827
|
+
"Use concrete file/symbol anchors when context exists.",
|
|
1828
|
+
"Avoid repeating generic process advice, broad deliverables lists, or organizational boilerplate."
|
|
1829
|
+
].join(" ");
|
|
1830
|
+
}
|
|
1831
|
+
function normalizeEnhancerContextPath(path) {
|
|
1832
|
+
return normalizePath(path).toLowerCase();
|
|
1833
|
+
}
|
|
1834
|
+
function looksLikeEnhancerConventionsFile(path) {
|
|
1835
|
+
const normalized = normalizeEnhancerContextPath(path);
|
|
1836
|
+
return (normalized === "agents.md" ||
|
|
1837
|
+
normalized.endsWith("/agents.md") ||
|
|
1838
|
+
normalized === "claude.md" ||
|
|
1839
|
+
normalized.endsWith("/claude.md") ||
|
|
1840
|
+
normalized === "readme.md" ||
|
|
1841
|
+
normalized.endsWith("/readme.md") ||
|
|
1842
|
+
normalized === "contributing.md" ||
|
|
1843
|
+
normalized.endsWith("/contributing.md"));
|
|
1844
|
+
}
|
|
1845
|
+
function extractProjectConventionsFromEnhancerContext(snippets) {
|
|
1846
|
+
const candidateSnippets = snippets.filter((snippet) => looksLikeEnhancerConventionsFile(snippet.path));
|
|
1847
|
+
if (candidateSnippets.length === 0) {
|
|
1848
|
+
return [];
|
|
1849
|
+
}
|
|
1850
|
+
const signalPattern = /\b(always|never|must|should|avoid|prefer|preserve|keep|strict|isolation|tenant|workspace|contract|schema|backward|compatibility|regression|test|typecheck|bun)\b/i;
|
|
1851
|
+
const out = [];
|
|
1852
|
+
const seen = new Set();
|
|
1853
|
+
for (const snippet of candidateSnippets) {
|
|
1854
|
+
const lines = snippet.snippet.split(/\r?\n/u);
|
|
1855
|
+
for (const rawLine of lines) {
|
|
1856
|
+
const cleaned = rawLine
|
|
1857
|
+
.replace(/^\s*[-*+]\s+/u, "")
|
|
1858
|
+
.replace(/^\s*\d+\.\s+/u, "")
|
|
1859
|
+
.trim();
|
|
1860
|
+
if (cleaned.length < 16 || cleaned.length > 180) {
|
|
1861
|
+
continue;
|
|
1862
|
+
}
|
|
1863
|
+
if (!signalPattern.test(cleaned)) {
|
|
1864
|
+
continue;
|
|
1865
|
+
}
|
|
1866
|
+
if (/^(import|export|const|let|var|if|for|while|return)\b/i.test(cleaned)) {
|
|
1867
|
+
continue;
|
|
1868
|
+
}
|
|
1869
|
+
const normalized = cleaned.toLowerCase();
|
|
1870
|
+
if (seen.has(normalized)) {
|
|
1871
|
+
continue;
|
|
1872
|
+
}
|
|
1873
|
+
seen.add(normalized);
|
|
1874
|
+
out.push(cleaned);
|
|
1875
|
+
if (out.length >= 8) {
|
|
1876
|
+
return out;
|
|
1877
|
+
}
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
return out;
|
|
1881
|
+
}
|
|
1882
|
+
function extractEnhancerNonNegotiables(input) {
|
|
1883
|
+
const combined = `${input.prompt}\n${input.history.map((entry) => entry.content).join("\n")}`;
|
|
1884
|
+
const lower = combined.toLowerCase();
|
|
1885
|
+
const out = [];
|
|
1886
|
+
const add = (value) => {
|
|
1887
|
+
if (!out.includes(value)) {
|
|
1888
|
+
out.push(value);
|
|
1889
|
+
}
|
|
1890
|
+
};
|
|
1891
|
+
if (/keep (?:behavior|behaviour) stable|preserve (?:existing )?(?:behavior|behaviour)|backward.?compat|no breaking changes|without breaking/i.test(lower)) {
|
|
1892
|
+
add("Preserve existing behavior and avoid breaking API/contract semantics.");
|
|
1893
|
+
}
|
|
1894
|
+
if (/regression tests?|add tests?|test coverage|boundary tests?/i.test(lower)) {
|
|
1895
|
+
add("Include regression tests for any changed behavior.");
|
|
1896
|
+
}
|
|
1897
|
+
if (/tenant|workspace|authorization|auth boundaries?|scope enforcement|isolation/i.test(lower)) {
|
|
1898
|
+
add("Maintain strict tenant/workspace isolation and authorization boundaries.");
|
|
1899
|
+
}
|
|
1900
|
+
if (/no docs|avoid docs|exclude docs/i.test(lower)) {
|
|
1901
|
+
add("Do not prioritize documentation-only changes unless explicitly requested.");
|
|
1902
|
+
}
|
|
1903
|
+
if (/no refactor|minimal changes?|smallest safe change/i.test(lower)) {
|
|
1904
|
+
add("Prefer the smallest safe change set.");
|
|
1905
|
+
}
|
|
1906
|
+
return out.slice(0, 6);
|
|
1907
|
+
}
|
|
1908
|
+
function buildEnhancerOutputContract(input) {
|
|
1909
|
+
const isConceptual = input.query_intent === "conceptual";
|
|
1910
|
+
if (input.style === "lean") {
|
|
1911
|
+
if (input.intent === "tests") {
|
|
1912
|
+
return {
|
|
1913
|
+
target_style: "lean_test_plan",
|
|
1914
|
+
max_words: input.has_context ? 220 : 170,
|
|
1915
|
+
preferred_sections: ["Goal", "Key test cases", "Validation"],
|
|
1916
|
+
avoid_patterns: ["long checklists", "broad architecture proposals", "generic deliverables blocks"]
|
|
1917
|
+
};
|
|
1918
|
+
}
|
|
1919
|
+
if (input.intent === "docs" || isConceptual) {
|
|
1920
|
+
return {
|
|
1921
|
+
target_style: "lean_spec",
|
|
1922
|
+
max_words: input.has_context ? 220 : 170,
|
|
1923
|
+
preferred_sections: ["Goal", "Scope", "Validation"],
|
|
1924
|
+
avoid_patterns: ["verbose outlines", "boilerplate context blocks", "generic process advice"]
|
|
1925
|
+
};
|
|
1926
|
+
}
|
|
1927
|
+
return {
|
|
1928
|
+
target_style: "lean_implementation_plan",
|
|
1929
|
+
max_words: input.has_context ? 230 : 180,
|
|
1930
|
+
preferred_sections: ["Goal", "Constraints", "Action steps", "Validation"],
|
|
1931
|
+
avoid_patterns: ["deep background sections", "broad deliverables lists", "repeated boilerplate"]
|
|
1932
|
+
};
|
|
1933
|
+
}
|
|
1934
|
+
if (input.style === "deep") {
|
|
1935
|
+
if (input.intent === "tests") {
|
|
1936
|
+
return {
|
|
1937
|
+
target_style: "deep_test_plan",
|
|
1938
|
+
max_words: input.has_context ? 420 : 340,
|
|
1939
|
+
preferred_sections: ["Goal", "Behavior under test", "Test matrix", "Edge cases", "Validation"],
|
|
1940
|
+
avoid_patterns: ["vague test advice", "non-test deliverables", "ungrounded file guesses"]
|
|
1941
|
+
};
|
|
1942
|
+
}
|
|
1943
|
+
if (input.intent === "docs" || isConceptual) {
|
|
1944
|
+
return {
|
|
1945
|
+
target_style: "deep_spec",
|
|
1946
|
+
max_words: input.has_context ? 420 : 340,
|
|
1947
|
+
preferred_sections: ["Goal", "Scope", "Relevant sources", "Proposed outline", "Risks", "Validation"],
|
|
1948
|
+
avoid_patterns: ["implementation-only checklists", "generic organizational boilerplate", "speculation"]
|
|
1949
|
+
};
|
|
1950
|
+
}
|
|
1951
|
+
return {
|
|
1952
|
+
target_style: "deep_implementation_plan",
|
|
1953
|
+
max_words: input.has_context ? 420 : 360,
|
|
1954
|
+
preferred_sections: [
|
|
1955
|
+
"Goal",
|
|
1956
|
+
"Scope and constraints",
|
|
1957
|
+
"Codebase anchors",
|
|
1958
|
+
"Implementation plan",
|
|
1959
|
+
"Edge cases",
|
|
1960
|
+
"Validation"
|
|
1961
|
+
],
|
|
1962
|
+
avoid_patterns: ["security theater", "repeated compliance boilerplate", "invented file/symbol references"]
|
|
1963
|
+
};
|
|
1964
|
+
}
|
|
1965
|
+
if (input.intent === "docs" || isConceptual) {
|
|
1966
|
+
return {
|
|
1967
|
+
target_style: "concise_spec",
|
|
1968
|
+
max_words: input.has_context ? 320 : 260,
|
|
1969
|
+
preferred_sections: ["Goal", "Scope", "Relevant sources", "Proposed outline", "Validation"],
|
|
1970
|
+
avoid_patterns: ["long implementation checklists", "generic deliverables sections", "repeated boilerplate"]
|
|
1971
|
+
};
|
|
1972
|
+
}
|
|
1973
|
+
if (input.intent === "tests") {
|
|
1974
|
+
return {
|
|
1975
|
+
target_style: "test_plan",
|
|
1976
|
+
max_words: input.has_context ? 320 : 260,
|
|
1977
|
+
preferred_sections: ["Goal", "Behavior under test", "Test matrix", "Implementation notes", "Validation"],
|
|
1978
|
+
avoid_patterns: ["broad architecture rewrites", "non-test deliverables", "generic process bullets"]
|
|
1979
|
+
};
|
|
1980
|
+
}
|
|
1981
|
+
return {
|
|
1982
|
+
target_style: "implementation_plan",
|
|
1983
|
+
max_words: input.has_context ? 360 : 300,
|
|
1984
|
+
preferred_sections: ["Goal", "Scope and constraints", "Codebase anchors", "Implementation plan", "Validation"],
|
|
1985
|
+
avoid_patterns: ["broad security theater", "repeated compliance boilerplate", "vague deliverables lists"]
|
|
1986
|
+
};
|
|
1987
|
+
}
|
|
1988
|
+
function buildClaudeEnhancerUserPayload(input) {
|
|
1989
|
+
const projectConventions = extractProjectConventionsFromEnhancerContext(input.context_snippets);
|
|
1990
|
+
const outputContract = buildEnhancerOutputContract({
|
|
1991
|
+
style: input.style_resolved,
|
|
1992
|
+
intent: input.intent,
|
|
1993
|
+
query_intent: input.query_intent,
|
|
1994
|
+
has_context: input.context_refs.length > 0
|
|
1995
|
+
});
|
|
1996
|
+
const nonNegotiables = extractEnhancerNonNegotiables({
|
|
1997
|
+
prompt: input.request.prompt,
|
|
1998
|
+
history: input.request.conversation_history
|
|
1999
|
+
});
|
|
2000
|
+
const payload = {
|
|
2001
|
+
trace_id: input.trace_id,
|
|
2002
|
+
tenant_id: input.tenant_id,
|
|
2003
|
+
workspace_id: input.workspace_id ?? "none",
|
|
2004
|
+
tool_mode: input.tool_mode,
|
|
2005
|
+
style_requested: input.style_requested,
|
|
2006
|
+
style_resolved: input.style_resolved,
|
|
2007
|
+
intent: input.intent,
|
|
2008
|
+
query_intent: input.query_intent,
|
|
2009
|
+
language: input.language,
|
|
2010
|
+
original_prompt: input.request.prompt,
|
|
2011
|
+
conversation_history: input.request.conversation_history,
|
|
2012
|
+
context_refs: input.context_refs,
|
|
2013
|
+
context_snippets: input.context_snippets.map((snippet) => ({
|
|
2014
|
+
path: snippet.path,
|
|
2015
|
+
start_line: snippet.start_line,
|
|
2016
|
+
end_line: snippet.end_line,
|
|
2017
|
+
reason: snippet.reason,
|
|
2018
|
+
score: Number(snippet.score.toFixed(4)),
|
|
2019
|
+
snippet: snippet.snippet
|
|
2020
|
+
})),
|
|
2021
|
+
output_contract: outputContract,
|
|
2022
|
+
non_negotiables: nonNegotiables,
|
|
2023
|
+
project_conventions: projectConventions
|
|
2024
|
+
};
|
|
2025
|
+
return [
|
|
2026
|
+
"Enhance the following request into a concise, implementation-ready prompt.",
|
|
2027
|
+
"Prioritize user intent fidelity, concrete repo anchors, and verifiable validation steps.",
|
|
2028
|
+
"Honor the requested enhancement style while avoiding invented details.",
|
|
2029
|
+
"Input JSON:",
|
|
2030
|
+
JSON.stringify(payload, null, 2)
|
|
2031
|
+
].join("\n");
|
|
2032
|
+
}
|
|
2033
|
+
function removeEnhancerCodeFences(text) {
|
|
2034
|
+
return text.trim().replace(/^```(?:json|markdown|md)?\s*/iu, "").replace(/\s*```$/u, "").trim();
|
|
2035
|
+
}
|
|
2036
|
+
function normalizeProviderEnhancedPrompt(text) {
|
|
2037
|
+
let normalized = removeEnhancerCodeFences(text).replace(/\r\n/g, "\n");
|
|
2038
|
+
normalized = normalized
|
|
2039
|
+
.split("\n")
|
|
2040
|
+
.map((line) => line.replace(/[ \t]+$/u, ""))
|
|
2041
|
+
.join("\n")
|
|
2042
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
2043
|
+
.trim();
|
|
2044
|
+
if (!normalized) {
|
|
2045
|
+
return normalized;
|
|
2046
|
+
}
|
|
2047
|
+
try {
|
|
2048
|
+
const payload = JSON.parse(normalized);
|
|
2049
|
+
if (payload && typeof payload === "object" && typeof payload.enhanced_prompt === "string") {
|
|
2050
|
+
return payload.enhanced_prompt.trim();
|
|
2051
|
+
}
|
|
2052
|
+
}
|
|
2053
|
+
catch {
|
|
2054
|
+
return normalized;
|
|
2055
|
+
}
|
|
2056
|
+
return normalized;
|
|
2057
|
+
}
|
|
2058
|
+
let cachedClaudeAgentSdkQueryFn;
|
|
2059
|
+
function isRecord(value) {
|
|
2060
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
2061
|
+
}
|
|
2062
|
+
async function loadClaudeAgentSdkQueryFn() {
|
|
2063
|
+
if (cachedClaudeAgentSdkQueryFn) {
|
|
2064
|
+
return cachedClaudeAgentSdkQueryFn;
|
|
2065
|
+
}
|
|
2066
|
+
const moduleNames = ["@anthropic-ai/claude-agent-sdk", "@anthropic-ai/claude-code"];
|
|
2067
|
+
let lastError;
|
|
2068
|
+
for (const moduleName of moduleNames) {
|
|
2069
|
+
try {
|
|
2070
|
+
const sdkModule = (await import(moduleName));
|
|
2071
|
+
if (typeof sdkModule.query === "function") {
|
|
2072
|
+
cachedClaudeAgentSdkQueryFn = sdkModule.query;
|
|
2073
|
+
return cachedClaudeAgentSdkQueryFn;
|
|
2074
|
+
}
|
|
2075
|
+
lastError = new Error(`${moduleName} does not export query()`);
|
|
2076
|
+
}
|
|
2077
|
+
catch (error) {
|
|
2078
|
+
lastError = error;
|
|
2079
|
+
}
|
|
2080
|
+
}
|
|
2081
|
+
const reason = lastError instanceof Error ? lastError.message : String(lastError ?? "unknown error");
|
|
2082
|
+
throw new EnhancerProviderRequestError("upstream_error", `claude agent sdk is not available; install @anthropic-ai/claude-agent-sdk (${reason})`);
|
|
2083
|
+
}
|
|
2084
|
+
function extractTextFromClaudeMessageContent(content) {
|
|
2085
|
+
if (typeof content === "string") {
|
|
2086
|
+
const trimmed = content.trim();
|
|
2087
|
+
return trimmed.length > 0 ? trimmed : undefined;
|
|
2088
|
+
}
|
|
2089
|
+
if (!Array.isArray(content)) {
|
|
2090
|
+
return undefined;
|
|
2091
|
+
}
|
|
2092
|
+
const parts = [];
|
|
2093
|
+
for (const item of content) {
|
|
2094
|
+
if (!isRecord(item)) {
|
|
2095
|
+
continue;
|
|
2096
|
+
}
|
|
2097
|
+
const text = item.text;
|
|
2098
|
+
if (typeof text !== "string") {
|
|
2099
|
+
continue;
|
|
2100
|
+
}
|
|
2101
|
+
const trimmed = text.trim();
|
|
2102
|
+
if (trimmed.length > 0) {
|
|
2103
|
+
parts.push(trimmed);
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
if (parts.length === 0) {
|
|
2107
|
+
return undefined;
|
|
2108
|
+
}
|
|
2109
|
+
return parts.join("\n");
|
|
2110
|
+
}
|
|
2111
|
+
function extractTextFromClaudeSdkMessage(message) {
|
|
2112
|
+
if (!isRecord(message)) {
|
|
2113
|
+
return undefined;
|
|
2114
|
+
}
|
|
2115
|
+
if (typeof message.summary === "string") {
|
|
2116
|
+
const trimmed = message.summary.trim();
|
|
2117
|
+
if (trimmed.length > 0) {
|
|
2118
|
+
return trimmed;
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
if (typeof message.result === "string") {
|
|
2122
|
+
const trimmed = message.result.trim();
|
|
2123
|
+
if (trimmed.length > 0) {
|
|
2124
|
+
return trimmed;
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
if (typeof message.text === "string") {
|
|
2128
|
+
const trimmed = message.text.trim();
|
|
2129
|
+
if (trimmed.length > 0) {
|
|
2130
|
+
return trimmed;
|
|
2131
|
+
}
|
|
2132
|
+
}
|
|
2133
|
+
const directContent = extractTextFromClaudeMessageContent(message.content);
|
|
2134
|
+
if (directContent) {
|
|
2135
|
+
return directContent;
|
|
2136
|
+
}
|
|
2137
|
+
if (isRecord(message.message)) {
|
|
2138
|
+
if (typeof message.message.text === "string") {
|
|
2139
|
+
const trimmed = message.message.text.trim();
|
|
2140
|
+
if (trimmed.length > 0) {
|
|
2141
|
+
return trimmed;
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
2144
|
+
const nestedContent = extractTextFromClaudeMessageContent(message.message.content);
|
|
2145
|
+
if (nestedContent) {
|
|
2146
|
+
return nestedContent;
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2149
|
+
return undefined;
|
|
2150
|
+
}
|
|
2151
|
+
function extractTextChunkFromClaudeSdkStreamEvent(message) {
|
|
2152
|
+
if (!isRecord(message) || message.type !== "stream_event") {
|
|
2153
|
+
return undefined;
|
|
2154
|
+
}
|
|
2155
|
+
const event = message.event;
|
|
2156
|
+
if (!isRecord(event)) {
|
|
2157
|
+
return undefined;
|
|
2158
|
+
}
|
|
2159
|
+
if (event.type === "content_block_start") {
|
|
2160
|
+
const contentBlock = event.content_block;
|
|
2161
|
+
if (isRecord(contentBlock) && typeof contentBlock.text === "string") {
|
|
2162
|
+
return contentBlock.text;
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
2165
|
+
if (event.type === "content_block_delta") {
|
|
2166
|
+
const delta = event.delta;
|
|
2167
|
+
if (!isRecord(delta)) {
|
|
2168
|
+
return undefined;
|
|
2169
|
+
}
|
|
2170
|
+
if (typeof delta.text === "string") {
|
|
2171
|
+
return delta.text;
|
|
2172
|
+
}
|
|
2173
|
+
}
|
|
2174
|
+
return undefined;
|
|
2175
|
+
}
|
|
2176
|
+
function extractStructuredOutputFromClaudeSdkMessage(message) {
|
|
2177
|
+
if (!isRecord(message)) {
|
|
2178
|
+
return undefined;
|
|
2179
|
+
}
|
|
2180
|
+
const structuredOutput = message.structured_output;
|
|
2181
|
+
if (!isRecord(structuredOutput)) {
|
|
2182
|
+
return undefined;
|
|
2183
|
+
}
|
|
2184
|
+
const enhancedPrompt = structuredOutput.enhanced_prompt;
|
|
2185
|
+
if (typeof enhancedPrompt !== "string" || enhancedPrompt.trim().length === 0) {
|
|
2186
|
+
return undefined;
|
|
2187
|
+
}
|
|
2188
|
+
return {
|
|
2189
|
+
enhanced_prompt: enhancedPrompt.trim()
|
|
2190
|
+
};
|
|
2191
|
+
}
|
|
2192
|
+
function extractResultFailureFromClaudeSdkMessage(message) {
|
|
2193
|
+
if (!isRecord(message) || message.type !== "result") {
|
|
2194
|
+
return undefined;
|
|
2195
|
+
}
|
|
2196
|
+
const subtype = message.subtype;
|
|
2197
|
+
if (typeof subtype !== "string" || subtype === "success") {
|
|
2198
|
+
return undefined;
|
|
2199
|
+
}
|
|
2200
|
+
const rawErrors = Array.isArray(message.errors) ? message.errors : [];
|
|
2201
|
+
const errors = rawErrors
|
|
2202
|
+
.filter((entry) => typeof entry === "string")
|
|
2203
|
+
.map((entry) => entry.trim())
|
|
2204
|
+
.filter((entry) => entry.length > 0);
|
|
2205
|
+
return {
|
|
2206
|
+
subtype,
|
|
2207
|
+
errors
|
|
2208
|
+
};
|
|
2209
|
+
}
|
|
2210
|
+
function describeClaudeSdkMessage(message) {
|
|
2211
|
+
if (!isRecord(message)) {
|
|
2212
|
+
return typeof message;
|
|
2213
|
+
}
|
|
2214
|
+
const type = typeof message.type === "string" ? message.type : "unknown";
|
|
2215
|
+
const subtype = typeof message.subtype === "string" ? message.subtype : undefined;
|
|
2216
|
+
return subtype ? `${type}:${subtype}` : type;
|
|
2217
|
+
}
|
|
2218
|
+
function classifyEnhancerProviderError(error) {
|
|
2219
|
+
if (error instanceof EnhancerProviderRequestError) {
|
|
2220
|
+
return error;
|
|
2221
|
+
}
|
|
2222
|
+
if (error instanceof Error) {
|
|
2223
|
+
const message = error.message || "unknown enhancer provider error";
|
|
2224
|
+
if (/(timeout|timed out|abort)/i.test(message)) {
|
|
2225
|
+
return new EnhancerProviderRequestError("timeout", message);
|
|
2226
|
+
}
|
|
2227
|
+
if (/(rate.?limit|too many requests|429)/i.test(message)) {
|
|
2228
|
+
return new EnhancerProviderRequestError("rate_limited", message);
|
|
2229
|
+
}
|
|
2230
|
+
if (/(no such file|not found|ENOENT)/i.test(message) && /claude/i.test(message)) {
|
|
2231
|
+
return new EnhancerProviderRequestError("upstream_error", `claude code executable not found: ${message}`);
|
|
2232
|
+
}
|
|
2233
|
+
return new EnhancerProviderRequestError("upstream_error", message);
|
|
2234
|
+
}
|
|
2235
|
+
return new EnhancerProviderRequestError("upstream_error", String(error));
|
|
2236
|
+
}
|
|
2237
|
+
export class ClaudeAgentEnhancerProvider {
|
|
2238
|
+
apiKey;
|
|
2239
|
+
model;
|
|
2240
|
+
maxTokens;
|
|
2241
|
+
baseUrl;
|
|
2242
|
+
pathToClaudeCodeExecutable;
|
|
2243
|
+
permissionMode;
|
|
2244
|
+
constructor(options) {
|
|
2245
|
+
const apiKey = options.api_key.trim();
|
|
2246
|
+
if (apiKey.length === 0) {
|
|
2247
|
+
throw new Error("invalid claude enhancer config: api_key must be non-empty");
|
|
2248
|
+
}
|
|
2249
|
+
const model = options.model?.trim() ?? DEFAULT_CLAUDE_ENHANCER_MODEL;
|
|
2250
|
+
if (model.length === 0) {
|
|
2251
|
+
throw new Error("invalid claude enhancer config: model must be non-empty");
|
|
2252
|
+
}
|
|
2253
|
+
const maxTokens = options.max_tokens ?? 1_200;
|
|
2254
|
+
if (!Number.isInteger(maxTokens) || maxTokens <= 0) {
|
|
2255
|
+
throw new Error("invalid claude enhancer config: max_tokens must be a positive integer");
|
|
2256
|
+
}
|
|
2257
|
+
const permissionMode = options.permission_mode ?? "default";
|
|
2258
|
+
if (permissionMode !== "default" &&
|
|
2259
|
+
permissionMode !== "acceptEdits" &&
|
|
2260
|
+
permissionMode !== "bypassPermissions" &&
|
|
2261
|
+
permissionMode !== "plan") {
|
|
2262
|
+
throw new Error("invalid claude enhancer config: permission_mode must be default|acceptEdits|bypassPermissions|plan");
|
|
2263
|
+
}
|
|
2264
|
+
this.apiKey = apiKey;
|
|
2265
|
+
this.model = model;
|
|
2266
|
+
this.maxTokens = maxTokens;
|
|
2267
|
+
this.baseUrl = options.base_url?.trim();
|
|
2268
|
+
const executablePath = options.path_to_claude_code_executable?.trim();
|
|
2269
|
+
this.pathToClaudeCodeExecutable = executablePath && executablePath.length > 0 ? executablePath : undefined;
|
|
2270
|
+
this.permissionMode = permissionMode;
|
|
2271
|
+
}
|
|
2272
|
+
describe() {
|
|
2273
|
+
return {
|
|
2274
|
+
provider: "claude_agent",
|
|
2275
|
+
model: this.model
|
|
2276
|
+
};
|
|
2277
|
+
}
|
|
2278
|
+
async generate(input) {
|
|
2279
|
+
const query = await loadClaudeAgentSdkQueryFn();
|
|
2280
|
+
const prompt = buildClaudeEnhancerUserPayload(input);
|
|
2281
|
+
const abortController = new AbortController();
|
|
2282
|
+
const upstreamAbortSignal = input.abort_signal;
|
|
2283
|
+
const upstreamAbortHandler = () => {
|
|
2284
|
+
abortController.abort();
|
|
2285
|
+
};
|
|
2286
|
+
if (upstreamAbortSignal) {
|
|
2287
|
+
if (upstreamAbortSignal.aborted) {
|
|
2288
|
+
abortController.abort();
|
|
2289
|
+
}
|
|
2290
|
+
else {
|
|
2291
|
+
upstreamAbortSignal.addEventListener("abort", upstreamAbortHandler, { once: true });
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
const options = {
|
|
2295
|
+
model: this.model,
|
|
2296
|
+
maxThinkingTokens: this.maxTokens,
|
|
2297
|
+
maxTurns: DEFAULT_CLAUDE_ENHANCER_MAX_TURNS,
|
|
2298
|
+
includePartialMessages: true,
|
|
2299
|
+
thinking: {
|
|
2300
|
+
type: "disabled"
|
|
2301
|
+
},
|
|
2302
|
+
permissionMode: this.permissionMode,
|
|
2303
|
+
systemPrompt: buildClaudeEnhancerSystemInstruction(input.language, input.style_resolved),
|
|
2304
|
+
// Enhancer already receives scoped context snippets; keep Claude Code tools disabled to avoid long tool loops.
|
|
2305
|
+
tools: [],
|
|
2306
|
+
allowedTools: [],
|
|
2307
|
+
env: {
|
|
2308
|
+
ANTHROPIC_API_KEY: this.apiKey,
|
|
2309
|
+
...(this.baseUrl ? { ANTHROPIC_BASE_URL: this.baseUrl } : {})
|
|
2310
|
+
},
|
|
2311
|
+
abortController,
|
|
2312
|
+
...(this.pathToClaudeCodeExecutable ? { pathToClaudeCodeExecutable: this.pathToClaudeCodeExecutable } : {}),
|
|
2313
|
+
...(input.request.project_root_path ? { cwd: input.request.project_root_path } : {})
|
|
2314
|
+
};
|
|
2315
|
+
let structured;
|
|
2316
|
+
let lastText;
|
|
2317
|
+
const streamTextParts = [];
|
|
2318
|
+
const seenMessageKinds = new Set();
|
|
2319
|
+
let maxTurnsFailure;
|
|
2320
|
+
try {
|
|
2321
|
+
for await (const message of query({ prompt, options })) {
|
|
2322
|
+
input.on_progress?.();
|
|
2323
|
+
seenMessageKinds.add(describeClaudeSdkMessage(message));
|
|
2324
|
+
const partialChunk = extractTextChunkFromClaudeSdkStreamEvent(message);
|
|
2325
|
+
if (typeof partialChunk === "string" && partialChunk.length > 0) {
|
|
2326
|
+
streamTextParts.push(partialChunk);
|
|
2327
|
+
}
|
|
2328
|
+
const resultFailure = extractResultFailureFromClaudeSdkMessage(message);
|
|
2329
|
+
if (resultFailure) {
|
|
2330
|
+
if (resultFailure.subtype === "error_max_turns") {
|
|
2331
|
+
maxTurnsFailure = resultFailure;
|
|
2332
|
+
continue;
|
|
2333
|
+
}
|
|
2334
|
+
const details = resultFailure.errors.length > 0 ? `: ${resultFailure.errors.join(" | ")}` : "";
|
|
2335
|
+
throw new EnhancerProviderRequestError("upstream_error", `claude agent sdk result error (${resultFailure.subtype})${details}`);
|
|
2336
|
+
}
|
|
2337
|
+
const maybeStructured = extractStructuredOutputFromClaudeSdkMessage(message);
|
|
2338
|
+
if (maybeStructured) {
|
|
2339
|
+
structured = maybeStructured;
|
|
2340
|
+
}
|
|
2341
|
+
const maybeText = extractTextFromClaudeSdkMessage(message);
|
|
2342
|
+
if (maybeText) {
|
|
2343
|
+
lastText = maybeText;
|
|
2344
|
+
}
|
|
2345
|
+
if (isRecord(message) && message.type === "assistant" && typeof message.error === "string") {
|
|
2346
|
+
throw new EnhancerProviderRequestError("upstream_error", `claude agent sdk assistant error: ${message.error}`);
|
|
2347
|
+
}
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
catch (error) {
|
|
2351
|
+
throw classifyEnhancerProviderError(error);
|
|
2352
|
+
}
|
|
2353
|
+
finally {
|
|
2354
|
+
if (upstreamAbortSignal) {
|
|
2355
|
+
upstreamAbortSignal.removeEventListener("abort", upstreamAbortHandler);
|
|
2356
|
+
}
|
|
2357
|
+
}
|
|
2358
|
+
if (structured) {
|
|
2359
|
+
return structured;
|
|
2360
|
+
}
|
|
2361
|
+
if (!lastText && streamTextParts.length > 0) {
|
|
2362
|
+
lastText = streamTextParts.join("").trim();
|
|
2363
|
+
}
|
|
2364
|
+
if (maxTurnsFailure && !lastText) {
|
|
2365
|
+
const details = maxTurnsFailure.errors.length > 0 ? `: ${maxTurnsFailure.errors.join(" | ")}` : "";
|
|
2366
|
+
throw new EnhancerProviderRequestError("upstream_error", `claude agent sdk hit max turns before returning output${details}`);
|
|
2367
|
+
}
|
|
2368
|
+
if (!lastText) {
|
|
2369
|
+
const seenKinds = [...seenMessageKinds].join(", ") || "none";
|
|
2370
|
+
throw new EnhancerProviderRequestError("invalid_response", `claude agent sdk returned no text output (messages=${seenKinds})`);
|
|
2371
|
+
}
|
|
2372
|
+
return { enhanced_prompt: normalizeProviderEnhancedPrompt(lastText) };
|
|
2373
|
+
}
|
|
2374
|
+
}
|
|
2375
|
+
async function safeResponseText(response) {
|
|
2376
|
+
try {
|
|
2377
|
+
const text = await response.text();
|
|
2378
|
+
return text.slice(0, 512);
|
|
2379
|
+
}
|
|
2380
|
+
catch {
|
|
2381
|
+
return "";
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
function parseRetryAfterMs(headerValue) {
|
|
2385
|
+
if (!headerValue) {
|
|
2386
|
+
return undefined;
|
|
2387
|
+
}
|
|
2388
|
+
const trimmed = headerValue.trim();
|
|
2389
|
+
if (!trimmed) {
|
|
2390
|
+
return undefined;
|
|
2391
|
+
}
|
|
2392
|
+
const seconds = Number(trimmed);
|
|
2393
|
+
if (Number.isFinite(seconds) && seconds >= 0) {
|
|
2394
|
+
return Math.ceil(seconds * 1000);
|
|
2395
|
+
}
|
|
2396
|
+
const dateMs = Date.parse(trimmed);
|
|
2397
|
+
if (!Number.isNaN(dateMs)) {
|
|
2398
|
+
return Math.max(0, dateMs - Date.now());
|
|
2399
|
+
}
|
|
2400
|
+
return undefined;
|
|
2401
|
+
}
|
|
2402
|
+
function resolveProviderLimiterScope(input) {
|
|
2403
|
+
const override = input.overrideScopeId?.trim();
|
|
2404
|
+
if (override) {
|
|
2405
|
+
return `provider:${input.provider}|credential:${override}`;
|
|
2406
|
+
}
|
|
2407
|
+
return `provider:${input.provider}|credential:${sha256(input.apiKey).slice(0, 16)}`;
|
|
2408
|
+
}
|
|
2409
|
+
function resolveEmbeddingDescriptor(provider) {
|
|
2410
|
+
const described = provider.describe?.();
|
|
2411
|
+
if (!described) {
|
|
2412
|
+
return {
|
|
2413
|
+
provider: "custom",
|
|
2414
|
+
dimensions: 24
|
|
2415
|
+
};
|
|
2416
|
+
}
|
|
2417
|
+
return {
|
|
2418
|
+
provider: described.provider,
|
|
2419
|
+
...(described.model ? { model: described.model } : {}),
|
|
2420
|
+
dimensions: described.dimensions,
|
|
2421
|
+
...(described.version ? { version: described.version } : {})
|
|
2422
|
+
};
|
|
2423
|
+
}
|
|
2424
|
+
function resolveRerankerDescriptor(provider) {
|
|
2425
|
+
const described = provider.describe?.();
|
|
2426
|
+
if (!described) {
|
|
2427
|
+
return {
|
|
2428
|
+
provider: "custom"
|
|
2429
|
+
};
|
|
2430
|
+
}
|
|
2431
|
+
return {
|
|
2432
|
+
provider: described.provider,
|
|
2433
|
+
...(described.model ? { model: described.model } : {})
|
|
2434
|
+
};
|
|
2435
|
+
}
|
|
2436
|
+
function resolveEnhancerProviderDescriptor(provider) {
|
|
2437
|
+
const described = provider.describe?.();
|
|
2438
|
+
if (!described) {
|
|
2439
|
+
return {
|
|
2440
|
+
provider: "custom"
|
|
2441
|
+
};
|
|
2442
|
+
}
|
|
2443
|
+
return {
|
|
2444
|
+
provider: described.provider,
|
|
2445
|
+
...(described.model ? { model: described.model } : {})
|
|
2446
|
+
};
|
|
2447
|
+
}
|
|
2448
|
+
function normalizeEmbeddingDescriptor(descriptor) {
|
|
2449
|
+
const provider = descriptor.provider.trim();
|
|
2450
|
+
if (provider.length === 0) {
|
|
2451
|
+
throw new Error("invalid embedding descriptor: provider must be non-empty");
|
|
2452
|
+
}
|
|
2453
|
+
if (!Number.isInteger(descriptor.dimensions) || descriptor.dimensions <= 0) {
|
|
2454
|
+
throw new Error("invalid embedding descriptor: dimensions must be a positive integer");
|
|
2455
|
+
}
|
|
2456
|
+
return {
|
|
2457
|
+
provider: provider.toLowerCase(),
|
|
2458
|
+
...(descriptor.model ? { model: descriptor.model.trim() } : {}),
|
|
1001
2459
|
dimensions: descriptor.dimensions,
|
|
1002
2460
|
...(descriptor.version ? { version: descriptor.version.trim() } : {})
|
|
1003
2461
|
};
|
|
1004
2462
|
}
|
|
2463
|
+
function normalizeRerankerDescriptor(descriptor) {
|
|
2464
|
+
const provider = descriptor.provider.trim().toLowerCase();
|
|
2465
|
+
if (provider.length === 0) {
|
|
2466
|
+
throw new Error("invalid reranker descriptor: provider must be non-empty");
|
|
2467
|
+
}
|
|
2468
|
+
const model = descriptor.model?.trim();
|
|
2469
|
+
return {
|
|
2470
|
+
provider,
|
|
2471
|
+
...(model ? { model } : {})
|
|
2472
|
+
};
|
|
2473
|
+
}
|
|
2474
|
+
function normalizeEnhancerProviderDescriptor(descriptor) {
|
|
2475
|
+
const provider = descriptor.provider.trim().toLowerCase();
|
|
2476
|
+
if (provider.length === 0) {
|
|
2477
|
+
throw new Error("invalid enhancer descriptor: provider must be non-empty");
|
|
2478
|
+
}
|
|
2479
|
+
const model = descriptor.model?.trim();
|
|
2480
|
+
return {
|
|
2481
|
+
provider,
|
|
2482
|
+
...(model ? { model } : {})
|
|
2483
|
+
};
|
|
2484
|
+
}
|
|
2485
|
+
function buildRerankerDocument(candidate) {
|
|
2486
|
+
return `${candidate.path}\n${candidate.snippet}`;
|
|
2487
|
+
}
|
|
2488
|
+
function classifyRerankerFailureReason(error) {
|
|
2489
|
+
if (error instanceof RerankerProviderRequestError) {
|
|
2490
|
+
if (error.reason === "timeout") {
|
|
2491
|
+
return "timeout";
|
|
2492
|
+
}
|
|
2493
|
+
if (error.reason === "rate_limited") {
|
|
2494
|
+
return "rate_limited";
|
|
2495
|
+
}
|
|
2496
|
+
if (error.reason === "invalid_json" || error.reason === "invalid_response") {
|
|
2497
|
+
return "schema_error";
|
|
2498
|
+
}
|
|
2499
|
+
return "upstream_error";
|
|
2500
|
+
}
|
|
2501
|
+
if (error instanceof Error) {
|
|
2502
|
+
if (/(rate.?limit|too many requests|429)/i.test(error.message)) {
|
|
2503
|
+
return "rate_limited";
|
|
2504
|
+
}
|
|
2505
|
+
if (/(timeout|timed out)/i.test(error.message)) {
|
|
2506
|
+
return "timeout";
|
|
2507
|
+
}
|
|
2508
|
+
return "upstream_error";
|
|
2509
|
+
}
|
|
2510
|
+
return "upstream_error";
|
|
2511
|
+
}
|
|
2512
|
+
function classifyEnhancerGenerationFailureReason(error) {
|
|
2513
|
+
if (error instanceof EnhancerProviderRequestError) {
|
|
2514
|
+
return error.reason;
|
|
2515
|
+
}
|
|
2516
|
+
if (error instanceof Error) {
|
|
2517
|
+
if (/(timeout|timed out)/i.test(error.message)) {
|
|
2518
|
+
return "timeout";
|
|
2519
|
+
}
|
|
2520
|
+
if (/(rate.?limit|too many requests|429)/i.test(error.message)) {
|
|
2521
|
+
return "rate_limited";
|
|
2522
|
+
}
|
|
2523
|
+
return "upstream_error";
|
|
2524
|
+
}
|
|
2525
|
+
return "upstream_error";
|
|
2526
|
+
}
|
|
1005
2527
|
function classifyIntent(prompt) {
|
|
1006
2528
|
const p = prompt.toLowerCase();
|
|
1007
2529
|
if (/fix|bug|error|crash|regression/.test(p)) {
|
|
@@ -1021,6 +2543,38 @@ function classifyIntent(prompt) {
|
|
|
1021
2543
|
}
|
|
1022
2544
|
return "unknown";
|
|
1023
2545
|
}
|
|
2546
|
+
function resolveEnhancerPromptStyle(input) {
|
|
2547
|
+
const requested = input.requested ?? "standard";
|
|
2548
|
+
if (requested !== "auto") {
|
|
2549
|
+
return {
|
|
2550
|
+
requested,
|
|
2551
|
+
resolved: requested
|
|
2552
|
+
};
|
|
2553
|
+
}
|
|
2554
|
+
const combined = `${input.prompt}\n${input.history.map((entry) => entry.content).join("\n")}`.trim();
|
|
2555
|
+
const words = tokenize(combined);
|
|
2556
|
+
const isShort = words.length <= 18 && input.history.length <= 1;
|
|
2557
|
+
const asksConcise = /\b(concise|brief|short|minimal|quick)\b/i.test(combined);
|
|
2558
|
+
const asksDepth = /\b(detailed|comprehensive|thorough|step-by-step|checklist)\b/i.test(combined);
|
|
2559
|
+
const highRisk = /\b(security|auth|authorization|tenant|workspace|migration|data loss|rollback|incident|compliance|backward)\b/i.test(combined);
|
|
2560
|
+
const complexityScore = Number(input.has_context) + Number(words.length >= 32) + Number(input.history.length >= 3);
|
|
2561
|
+
if (asksConcise || (isShort && !highRisk && !asksDepth)) {
|
|
2562
|
+
return {
|
|
2563
|
+
requested,
|
|
2564
|
+
resolved: "lean"
|
|
2565
|
+
};
|
|
2566
|
+
}
|
|
2567
|
+
if (asksDepth || highRisk || complexityScore >= 2 || input.query_intent === "symbol-heavy" || input.intent === "tests") {
|
|
2568
|
+
return {
|
|
2569
|
+
requested,
|
|
2570
|
+
resolved: "deep"
|
|
2571
|
+
};
|
|
2572
|
+
}
|
|
2573
|
+
return {
|
|
2574
|
+
requested,
|
|
2575
|
+
resolved: "standard"
|
|
2576
|
+
};
|
|
2577
|
+
}
|
|
1024
2578
|
function detectDominantLanguage(prompt, history) {
|
|
1025
2579
|
const latestUser = [...history].reverse().find((m) => m.role === "user")?.content ?? prompt;
|
|
1026
2580
|
const sample = `${prompt}\n${latestUser}`.toLowerCase();
|
|
@@ -1464,7 +3018,7 @@ function buildEnhancerRetrievalQuery(prompt, history, options) {
|
|
|
1464
3018
|
query_intent: queryIntent
|
|
1465
3019
|
};
|
|
1466
3020
|
}
|
|
1467
|
-
const ENHANCER_LOW_CONFIDENCE_WARNING = "Low retrieval confidence; narrowed context refs
|
|
3021
|
+
const ENHANCER_LOW_CONFIDENCE_WARNING = "Low retrieval confidence; narrowed context refs.";
|
|
1468
3022
|
const ENHANCER_CONFIDENCE_OVERLAP_STOPWORDS = new Set([
|
|
1469
3023
|
"a",
|
|
1470
3024
|
"about",
|
|
@@ -1612,38 +3166,725 @@ function isRiskyEnhancerPath(path, intent) {
|
|
|
1612
3166
|
}
|
|
1613
3167
|
return isDocsLikePath(path) || isTestLikePath(path) || isExampleLikePath(path);
|
|
1614
3168
|
}
|
|
1615
|
-
function applyEnhancerIntentPathFiltering(results, input) {
|
|
1616
|
-
if (results.length === 0) {
|
|
1617
|
-
return [];
|
|
3169
|
+
function applyEnhancerIntentPathFiltering(results, input) {
|
|
3170
|
+
if (results.length === 0) {
|
|
3171
|
+
return [];
|
|
3172
|
+
}
|
|
3173
|
+
const preferred = results.filter((result) => !isRiskyEnhancerPath(result.path, input.intent) && !shouldAvoidPathFromNegation(result.path, input.negative_preferences));
|
|
3174
|
+
if (preferred.length > 0) {
|
|
3175
|
+
return preferred;
|
|
3176
|
+
}
|
|
3177
|
+
if (input.strict_impl_only_filtering) {
|
|
3178
|
+
const implOnly = results.filter((result) => !isDocsLikePath(result.path) &&
|
|
3179
|
+
!isTestLikePath(result.path) &&
|
|
3180
|
+
!isExampleLikePath(result.path) &&
|
|
3181
|
+
!isArchiveLikePath(result.path));
|
|
3182
|
+
if (implOnly.length > 0) {
|
|
3183
|
+
return implOnly;
|
|
3184
|
+
}
|
|
3185
|
+
}
|
|
3186
|
+
const tolerated = results.filter((result) => !shouldAvoidPathFromNegation(result.path, input.negative_preferences));
|
|
3187
|
+
return tolerated.length > 0 ? tolerated : results;
|
|
3188
|
+
}
|
|
3189
|
+
function compareSearchResults(a, b) {
|
|
3190
|
+
const scoreDiff = b.score - a.score;
|
|
3191
|
+
if (Math.abs(scoreDiff) > 1e-9) {
|
|
3192
|
+
return scoreDiff;
|
|
3193
|
+
}
|
|
3194
|
+
if (a.path !== b.path) {
|
|
3195
|
+
return a.path.localeCompare(b.path);
|
|
3196
|
+
}
|
|
3197
|
+
if (a.start_line !== b.start_line) {
|
|
3198
|
+
return a.start_line - b.start_line;
|
|
3199
|
+
}
|
|
3200
|
+
return a.end_line - b.end_line;
|
|
3201
|
+
}
|
|
3202
|
+
function compareSearchResultsByLineRange(a, b) {
|
|
3203
|
+
if (a.start_line !== b.start_line) {
|
|
3204
|
+
return a.start_line - b.start_line;
|
|
3205
|
+
}
|
|
3206
|
+
if (a.end_line !== b.end_line) {
|
|
3207
|
+
return a.end_line - b.end_line;
|
|
3208
|
+
}
|
|
3209
|
+
return compareSearchResults(a, b);
|
|
3210
|
+
}
|
|
3211
|
+
function mergeSnippetCluster(cluster, mergedStartLine, mergedEndLine) {
|
|
3212
|
+
const byRelevance = [...cluster].sort(compareSearchResults);
|
|
3213
|
+
const primary = byRelevance[0];
|
|
3214
|
+
if (!primary) {
|
|
3215
|
+
return "";
|
|
3216
|
+
}
|
|
3217
|
+
const lineMap = new Map();
|
|
3218
|
+
for (let rank = 0; rank < byRelevance.length; rank += 1) {
|
|
3219
|
+
const candidate = byRelevance[rank];
|
|
3220
|
+
if (!candidate) {
|
|
3221
|
+
continue;
|
|
3222
|
+
}
|
|
3223
|
+
const lines = candidate.snippet.replace(/\r\n/g, "\n").split("\n");
|
|
3224
|
+
const expectedLineCount = Math.max(1, candidate.end_line - candidate.start_line + 1);
|
|
3225
|
+
const maxLines = Math.min(lines.length, expectedLineCount);
|
|
3226
|
+
for (let offset = 0; offset < maxLines; offset += 1) {
|
|
3227
|
+
const lineNumber = candidate.start_line + offset;
|
|
3228
|
+
if (lineNumber < mergedStartLine || lineNumber > mergedEndLine) {
|
|
3229
|
+
continue;
|
|
3230
|
+
}
|
|
3231
|
+
const text = lines[offset];
|
|
3232
|
+
if (typeof text !== "string") {
|
|
3233
|
+
continue;
|
|
3234
|
+
}
|
|
3235
|
+
const existing = lineMap.get(lineNumber);
|
|
3236
|
+
if (!existing || candidate.score > existing.score + 1e-9 || (Math.abs(candidate.score - existing.score) <= 1e-9 && rank < existing.rank)) {
|
|
3237
|
+
lineMap.set(lineNumber, { text, score: candidate.score, rank });
|
|
3238
|
+
}
|
|
3239
|
+
}
|
|
3240
|
+
}
|
|
3241
|
+
const mergedLines = [];
|
|
3242
|
+
let missingLines = 0;
|
|
3243
|
+
for (let line = mergedStartLine; line <= mergedEndLine; line += 1) {
|
|
3244
|
+
const entry = lineMap.get(line);
|
|
3245
|
+
if (!entry) {
|
|
3246
|
+
missingLines += 1;
|
|
3247
|
+
mergedLines.push("");
|
|
3248
|
+
continue;
|
|
3249
|
+
}
|
|
3250
|
+
mergedLines.push(entry.text);
|
|
3251
|
+
}
|
|
3252
|
+
const totalLines = Math.max(1, mergedEndLine - mergedStartLine + 1);
|
|
3253
|
+
const maxMissingLines = Math.max(2, Math.floor(totalLines * 0.2));
|
|
3254
|
+
if (missingLines > maxMissingLines) {
|
|
3255
|
+
return primary.snippet;
|
|
3256
|
+
}
|
|
3257
|
+
return mergedLines.join("\n");
|
|
3258
|
+
}
|
|
3259
|
+
function mergeCandidateCluster(cluster) {
|
|
3260
|
+
if (cluster.length === 0) {
|
|
3261
|
+
throw new Error("mergeCandidateCluster requires at least one candidate");
|
|
3262
|
+
}
|
|
3263
|
+
if (cluster.length === 1) {
|
|
3264
|
+
return cluster[0];
|
|
3265
|
+
}
|
|
3266
|
+
const byRelevance = [...cluster].sort(compareSearchResults);
|
|
3267
|
+
const primary = byRelevance[0];
|
|
3268
|
+
const mergedStartLine = Math.min(...cluster.map((candidate) => candidate.start_line));
|
|
3269
|
+
const mergedEndLine = Math.max(...cluster.map((candidate) => candidate.end_line));
|
|
3270
|
+
const stitchedSnippet = mergeSnippetCluster(cluster, mergedStartLine, mergedEndLine);
|
|
3271
|
+
return {
|
|
3272
|
+
...primary,
|
|
3273
|
+
start_line: mergedStartLine,
|
|
3274
|
+
end_line: mergedEndLine,
|
|
3275
|
+
snippet: stitchedSnippet.length > 0 ? stitchedSnippet : primary.snippet
|
|
3276
|
+
};
|
|
3277
|
+
}
|
|
3278
|
+
const HEAVY_LINE_RANGE_OVERLAP_RATIO = 0.2;
|
|
3279
|
+
function lineRangeLength(startLine, endLine) {
|
|
3280
|
+
return Math.max(1, endLine - startLine + 1);
|
|
3281
|
+
}
|
|
3282
|
+
function lineRangeOverlapLength(aStartLine, aEndLine, bStartLine, bEndLine) {
|
|
3283
|
+
const start = Math.max(aStartLine, bStartLine);
|
|
3284
|
+
const end = Math.min(aEndLine, bEndLine);
|
|
3285
|
+
if (end < start) {
|
|
3286
|
+
return 0;
|
|
3287
|
+
}
|
|
3288
|
+
return end - start + 1;
|
|
3289
|
+
}
|
|
3290
|
+
function isHeavilyOverlappingLineRange(candidate, selectedRanges) {
|
|
3291
|
+
for (const selected of selectedRanges) {
|
|
3292
|
+
const overlapLength = lineRangeOverlapLength(selected.start_line, selected.end_line, candidate.start_line, candidate.end_line);
|
|
3293
|
+
if (overlapLength <= 0) {
|
|
3294
|
+
continue;
|
|
3295
|
+
}
|
|
3296
|
+
const smallerRange = Math.min(lineRangeLength(selected.start_line, selected.end_line), lineRangeLength(candidate.start_line, candidate.end_line));
|
|
3297
|
+
const overlapRatio = overlapLength / Math.max(1, smallerRange);
|
|
3298
|
+
if (overlapRatio >= HEAVY_LINE_RANGE_OVERLAP_RATIO) {
|
|
3299
|
+
return true;
|
|
3300
|
+
}
|
|
3301
|
+
}
|
|
3302
|
+
return false;
|
|
3303
|
+
}
|
|
3304
|
+
function mergeLineSpans(spans) {
|
|
3305
|
+
if (spans.length <= 1) {
|
|
3306
|
+
return [...spans];
|
|
3307
|
+
}
|
|
3308
|
+
const ordered = [...spans]
|
|
3309
|
+
.filter((span) => span.end_line >= span.start_line)
|
|
3310
|
+
.sort((a, b) => a.start_line - b.start_line || a.end_line - b.end_line);
|
|
3311
|
+
const merged = [];
|
|
3312
|
+
for (const span of ordered) {
|
|
3313
|
+
const last = merged[merged.length - 1];
|
|
3314
|
+
if (!last || span.start_line > last.end_line + 1) {
|
|
3315
|
+
merged.push({ ...span });
|
|
3316
|
+
continue;
|
|
3317
|
+
}
|
|
3318
|
+
last.end_line = Math.max(last.end_line, span.end_line);
|
|
3319
|
+
}
|
|
3320
|
+
return merged;
|
|
3321
|
+
}
|
|
3322
|
+
function lineRangeGap(anchor, candidate) {
|
|
3323
|
+
if (candidate.start_line > anchor.end_line) {
|
|
3324
|
+
return candidate.start_line - anchor.end_line - 1;
|
|
3325
|
+
}
|
|
3326
|
+
if (anchor.start_line > candidate.end_line) {
|
|
3327
|
+
return anchor.start_line - candidate.end_line - 1;
|
|
3328
|
+
}
|
|
3329
|
+
return 0;
|
|
3330
|
+
}
|
|
3331
|
+
function buildPreferredLineMap(candidates) {
|
|
3332
|
+
const byRelevance = [...candidates].sort(compareSearchResults);
|
|
3333
|
+
const lineMap = new Map();
|
|
3334
|
+
for (let rank = 0; rank < byRelevance.length; rank += 1) {
|
|
3335
|
+
const candidate = byRelevance[rank];
|
|
3336
|
+
if (!candidate) {
|
|
3337
|
+
continue;
|
|
3338
|
+
}
|
|
3339
|
+
const lines = candidate.snippet.replace(/\r\n/g, "\n").split("\n");
|
|
3340
|
+
const expectedLineCount = Math.max(1, candidate.end_line - candidate.start_line + 1);
|
|
3341
|
+
const maxLines = Math.min(lines.length, expectedLineCount);
|
|
3342
|
+
for (let offset = 0; offset < maxLines; offset += 1) {
|
|
3343
|
+
const lineNumber = candidate.start_line + offset;
|
|
3344
|
+
const text = lines[offset];
|
|
3345
|
+
if (typeof text !== "string") {
|
|
3346
|
+
continue;
|
|
3347
|
+
}
|
|
3348
|
+
const existing = lineMap.get(lineNumber);
|
|
3349
|
+
if (!existing || candidate.score > existing.score + 1e-9 || (Math.abs(candidate.score - existing.score) <= 1e-9 && rank < existing.rank)) {
|
|
3350
|
+
lineMap.set(lineNumber, { text, score: candidate.score, rank });
|
|
3351
|
+
}
|
|
3352
|
+
}
|
|
3353
|
+
}
|
|
3354
|
+
return new Map([...lineMap.entries()].map(([line, value]) => [line, value.text]));
|
|
3355
|
+
}
|
|
3356
|
+
function clipSnippetToMaxChars(snippet, maxChars) {
|
|
3357
|
+
if (snippet.length <= maxChars) {
|
|
3358
|
+
return snippet;
|
|
3359
|
+
}
|
|
3360
|
+
const clipped = snippet.slice(0, Math.max(0, maxChars));
|
|
3361
|
+
const lastNewline = clipped.lastIndexOf("\n");
|
|
3362
|
+
if (lastNewline > 80) {
|
|
3363
|
+
return clipped.slice(0, lastNewline).trimEnd();
|
|
3364
|
+
}
|
|
3365
|
+
return clipped.trimEnd();
|
|
3366
|
+
}
|
|
3367
|
+
function snippetIntegrityLanguageFromPath(path) {
|
|
3368
|
+
const extension = fileExtension(path);
|
|
3369
|
+
if (extension === ".ts" || extension === ".mts" || extension === ".cts") {
|
|
3370
|
+
return "typescript";
|
|
3371
|
+
}
|
|
3372
|
+
if (extension === ".tsx") {
|
|
3373
|
+
return "tsx";
|
|
3374
|
+
}
|
|
3375
|
+
if (extension === ".js" || extension === ".mjs" || extension === ".cjs") {
|
|
3376
|
+
return "javascript";
|
|
3377
|
+
}
|
|
3378
|
+
if (extension === ".jsx") {
|
|
3379
|
+
return "jsx";
|
|
3380
|
+
}
|
|
3381
|
+
return undefined;
|
|
3382
|
+
}
|
|
3383
|
+
function firstNonEmptyLine(snippet) {
|
|
3384
|
+
const lines = snippet.replace(/\r\n/g, "\n").split("\n");
|
|
3385
|
+
for (const line of lines) {
|
|
3386
|
+
const trimmed = line.trim();
|
|
3387
|
+
if (trimmed.length > 0) {
|
|
3388
|
+
return trimmed;
|
|
3389
|
+
}
|
|
3390
|
+
}
|
|
3391
|
+
return "";
|
|
3392
|
+
}
|
|
3393
|
+
function lastNonEmptyLine(snippet) {
|
|
3394
|
+
const lines = snippet.replace(/\r\n/g, "\n").split("\n");
|
|
3395
|
+
for (let idx = lines.length - 1; idx >= 0; idx -= 1) {
|
|
3396
|
+
const trimmed = (lines[idx] ?? "").trim();
|
|
3397
|
+
if (trimmed.length > 0) {
|
|
3398
|
+
return trimmed;
|
|
3399
|
+
}
|
|
3400
|
+
}
|
|
3401
|
+
return "";
|
|
3402
|
+
}
|
|
3403
|
+
function curlyBraceDelta(snippet) {
|
|
3404
|
+
let opens = 0;
|
|
3405
|
+
let closes = 0;
|
|
3406
|
+
for (const char of snippet) {
|
|
3407
|
+
if (char === "{") {
|
|
3408
|
+
opens += 1;
|
|
3409
|
+
continue;
|
|
3410
|
+
}
|
|
3411
|
+
if (char === "}") {
|
|
3412
|
+
closes += 1;
|
|
3413
|
+
}
|
|
3414
|
+
}
|
|
3415
|
+
return opens - closes;
|
|
3416
|
+
}
|
|
3417
|
+
function looksLikeDeclarationStart(line) {
|
|
3418
|
+
if (line.length === 0) {
|
|
3419
|
+
return false;
|
|
3420
|
+
}
|
|
3421
|
+
if (line.startsWith("@")) {
|
|
3422
|
+
return true;
|
|
3423
|
+
}
|
|
3424
|
+
return (/^(?:export\s+)?(?:async\s+)?function\s+[A-Za-z_$][\w$]*\s*\(/u.test(line) ||
|
|
3425
|
+
/^(?:export\s+)?(?:default\s+)?class\s+[A-Za-z_$][\w$]*/u.test(line) ||
|
|
3426
|
+
/^(?:export\s+)?(?:const|let|var)\s+[A-Za-z_$][\w$]*\s*=/u.test(line) ||
|
|
3427
|
+
/^(?:public|private|protected|static|readonly|async)\s+[A-Za-z_$][\w$]*\s*\(/u.test(line) ||
|
|
3428
|
+
/^(?:[A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{/u.test(line));
|
|
3429
|
+
}
|
|
3430
|
+
function looksLikeSnippetTerminalBoundary(line) {
|
|
3431
|
+
if (line.length === 0) {
|
|
3432
|
+
return false;
|
|
3433
|
+
}
|
|
3434
|
+
return (line.endsWith("}") ||
|
|
3435
|
+
line.endsWith("};") ||
|
|
3436
|
+
line.endsWith(");") ||
|
|
3437
|
+
line.endsWith("]") ||
|
|
3438
|
+
line.endsWith("];"));
|
|
3439
|
+
}
|
|
3440
|
+
function detectSnippetSymbolName(snippet) {
|
|
3441
|
+
const lines = snippet.replace(/\r\n/g, "\n").split("\n").slice(0, 40);
|
|
3442
|
+
const patterns = [
|
|
3443
|
+
/^(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\(/u,
|
|
3444
|
+
/^(?:export\s+)?(?:default\s+)?class\s+([A-Za-z_$][\w$]*)\b/u,
|
|
3445
|
+
/^(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\([^)]*\)\s*=>/u,
|
|
3446
|
+
/^(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?[A-Za-z_$][\w$]*\s*=>/u,
|
|
3447
|
+
/^(?:public|private|protected|static|readonly|async)\s+([A-Za-z_$][\w$]*)\s*\(/u,
|
|
3448
|
+
/^([A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{/u
|
|
3449
|
+
];
|
|
3450
|
+
const disallowed = new Set(["if", "for", "while", "switch", "catch", "return"]);
|
|
3451
|
+
for (const line of lines) {
|
|
3452
|
+
const trimmed = line.trim();
|
|
3453
|
+
if (trimmed.length === 0) {
|
|
3454
|
+
continue;
|
|
3455
|
+
}
|
|
3456
|
+
for (const pattern of patterns) {
|
|
3457
|
+
const match = trimmed.match(pattern);
|
|
3458
|
+
const symbol = match?.[1];
|
|
3459
|
+
if (symbol && !disallowed.has(symbol)) {
|
|
3460
|
+
return symbol;
|
|
3461
|
+
}
|
|
3462
|
+
}
|
|
3463
|
+
}
|
|
3464
|
+
return undefined;
|
|
3465
|
+
}
|
|
3466
|
+
function shouldAnnotateSnippetAsTruncated(result, omittedBefore, omittedAfter) {
|
|
3467
|
+
if (omittedBefore <= 0 && omittedAfter <= 0) {
|
|
3468
|
+
return false;
|
|
3469
|
+
}
|
|
3470
|
+
const firstLine = firstNonEmptyLine(result.snippet);
|
|
3471
|
+
const lastLine = lastNonEmptyLine(result.snippet);
|
|
3472
|
+
if (omittedBefore > 0 && !looksLikeDeclarationStart(firstLine)) {
|
|
3473
|
+
return true;
|
|
3474
|
+
}
|
|
3475
|
+
if (omittedAfter > 0) {
|
|
3476
|
+
if (curlyBraceDelta(result.snippet) > 0) {
|
|
3477
|
+
return true;
|
|
3478
|
+
}
|
|
3479
|
+
if (!looksLikeSnippetTerminalBoundary(lastLine)) {
|
|
3480
|
+
return true;
|
|
3481
|
+
}
|
|
3482
|
+
}
|
|
3483
|
+
return omittedBefore > 0 && omittedAfter > 0;
|
|
3484
|
+
}
|
|
3485
|
+
function estimateContiguousEnvelope(input) {
|
|
3486
|
+
let start = input.anchor.start_line;
|
|
3487
|
+
let end = input.anchor.end_line;
|
|
3488
|
+
let changed = true;
|
|
3489
|
+
while (changed) {
|
|
3490
|
+
changed = false;
|
|
3491
|
+
for (const candidate of input.candidates) {
|
|
3492
|
+
const gap = lineRangeGap({ start_line: start, end_line: end }, candidate);
|
|
3493
|
+
if (gap > input.maxGapLines) {
|
|
3494
|
+
continue;
|
|
3495
|
+
}
|
|
3496
|
+
const nextStart = Math.min(start, candidate.start_line);
|
|
3497
|
+
const nextEnd = Math.max(end, candidate.end_line);
|
|
3498
|
+
if (nextStart !== start || nextEnd !== end) {
|
|
3499
|
+
start = nextStart;
|
|
3500
|
+
end = nextEnd;
|
|
3501
|
+
changed = true;
|
|
3502
|
+
}
|
|
3503
|
+
}
|
|
3504
|
+
}
|
|
3505
|
+
return { start_line: start, end_line: end };
|
|
3506
|
+
}
|
|
3507
|
+
function repairSnippetFromEnvelope(input) {
|
|
3508
|
+
const envelopeSpan = input.envelope.end_line - input.envelope.start_line + 1;
|
|
3509
|
+
if (envelopeSpan > input.config.repair_max_envelope_lines) {
|
|
3510
|
+
return { reason: "envelope_cap_exceeded", clipped: false };
|
|
3511
|
+
}
|
|
3512
|
+
const envelopeCandidates = input.samePathCandidates
|
|
3513
|
+
.filter((candidate) => candidate.end_line >= input.envelope.start_line && candidate.start_line <= input.envelope.end_line)
|
|
3514
|
+
.sort(compareSearchResultsByLineRange);
|
|
3515
|
+
if (envelopeCandidates.length === 0) {
|
|
3516
|
+
return { reason: "no_envelope_candidates", clipped: false };
|
|
3517
|
+
}
|
|
3518
|
+
const lineMap = buildPreferredLineMap(envelopeCandidates);
|
|
3519
|
+
const renderedLines = [];
|
|
3520
|
+
let missingLines = 0;
|
|
3521
|
+
for (let line = input.envelope.start_line; line <= input.envelope.end_line; line += 1) {
|
|
3522
|
+
const text = lineMap.get(line);
|
|
3523
|
+
if (typeof text !== "string") {
|
|
3524
|
+
missingLines += 1;
|
|
3525
|
+
renderedLines.push("");
|
|
3526
|
+
continue;
|
|
3527
|
+
}
|
|
3528
|
+
renderedLines.push(text);
|
|
3529
|
+
}
|
|
3530
|
+
const maxMissingLines = Math.max(2, Math.floor(envelopeSpan * 0.2));
|
|
3531
|
+
if (missingLines > maxMissingLines) {
|
|
3532
|
+
return { reason: "missing_line_density_too_high", clipped: false };
|
|
3533
|
+
}
|
|
3534
|
+
const clippedLines = [];
|
|
3535
|
+
let usedChars = 0;
|
|
3536
|
+
let clipped = false;
|
|
3537
|
+
for (let index = 0; index < renderedLines.length; index += 1) {
|
|
3538
|
+
const line = renderedLines[index] ?? "";
|
|
3539
|
+
const additionalChars = index === 0 ? line.length : line.length + 1;
|
|
3540
|
+
if (clippedLines.length > 0 && usedChars + additionalChars > input.config.repair_max_snippet_chars) {
|
|
3541
|
+
clipped = true;
|
|
3542
|
+
break;
|
|
3543
|
+
}
|
|
3544
|
+
if (clippedLines.length === 0 && line.length > input.config.repair_max_snippet_chars) {
|
|
3545
|
+
const clippedLine = line.slice(0, input.config.repair_max_snippet_chars);
|
|
3546
|
+
if (clippedLine.length === 0) {
|
|
3547
|
+
return { reason: "snippet_char_cap_exceeded", clipped: false };
|
|
3548
|
+
}
|
|
3549
|
+
clippedLines.push(clippedLine);
|
|
3550
|
+
usedChars = clippedLine.length;
|
|
3551
|
+
clipped = true;
|
|
3552
|
+
break;
|
|
3553
|
+
}
|
|
3554
|
+
clippedLines.push(line);
|
|
3555
|
+
usedChars += additionalChars;
|
|
3556
|
+
}
|
|
3557
|
+
if (clippedLines.length === 0) {
|
|
3558
|
+
return { reason: "snippet_char_cap_exceeded", clipped: false };
|
|
3559
|
+
}
|
|
3560
|
+
const repairedSnippet = clippedLines.join("\n").trimEnd();
|
|
3561
|
+
if (repairedSnippet.length === 0) {
|
|
3562
|
+
return { reason: "empty_repaired_snippet", clipped: false };
|
|
3563
|
+
}
|
|
3564
|
+
const repairedEndLine = input.envelope.start_line + clippedLines.length - 1;
|
|
3565
|
+
return {
|
|
3566
|
+
repaired: {
|
|
3567
|
+
...input.anchor,
|
|
3568
|
+
start_line: input.envelope.start_line,
|
|
3569
|
+
end_line: repairedEndLine,
|
|
3570
|
+
snippet: repairedSnippet
|
|
3571
|
+
},
|
|
3572
|
+
clipped
|
|
3573
|
+
};
|
|
3574
|
+
}
|
|
3575
|
+
function buildSnippetTruncationMarker(input) {
|
|
3576
|
+
const estimatedTotalLines = Math.max(1, input.envelope_end_line - input.envelope_start_line + 1);
|
|
3577
|
+
const omittedBefore = Math.max(0, input.result.start_line - input.envelope_start_line);
|
|
3578
|
+
const omittedAfter = Math.max(0, input.envelope_end_line - input.result.end_line);
|
|
3579
|
+
return `// [truncated:${input.marker_template_version} symbol=${input.symbolName ?? "unknown"} estimated_span=${input.envelope_start_line}-${input.envelope_end_line} estimated_total_lines=${estimatedTotalLines} omitted_before=${omittedBefore} omitted_after=${omittedAfter} through_line=${input.result.end_line}]`;
|
|
3580
|
+
}
|
|
3581
|
+
function annotateSearchResultsWithSnippetIntegrity(input) {
|
|
3582
|
+
if (!input.config.enabled || input.selected.length === 0) {
|
|
3583
|
+
return [...input.selected];
|
|
3584
|
+
}
|
|
3585
|
+
const enabledLanguages = new Set(normalizeSnippetIntegrityLanguageList(input.config.target_languages));
|
|
3586
|
+
if (enabledLanguages.size === 0) {
|
|
3587
|
+
return [...input.selected];
|
|
3588
|
+
}
|
|
3589
|
+
const sourceByPath = new Map();
|
|
3590
|
+
for (const candidate of input.sourceCandidates) {
|
|
3591
|
+
const rows = sourceByPath.get(candidate.path);
|
|
3592
|
+
if (rows) {
|
|
3593
|
+
rows.push(candidate);
|
|
3594
|
+
}
|
|
3595
|
+
else {
|
|
3596
|
+
sourceByPath.set(candidate.path, [candidate]);
|
|
3597
|
+
}
|
|
3598
|
+
}
|
|
3599
|
+
return input.selected.map((result) => {
|
|
3600
|
+
const language = snippetIntegrityLanguageFromPath(result.path);
|
|
3601
|
+
if (!language || !enabledLanguages.has(language)) {
|
|
3602
|
+
return result;
|
|
3603
|
+
}
|
|
3604
|
+
const samePath = sourceByPath.get(result.path) ?? [result];
|
|
3605
|
+
if (samePath.length <= 1) {
|
|
3606
|
+
return result;
|
|
3607
|
+
}
|
|
3608
|
+
const envelope = estimateContiguousEnvelope({
|
|
3609
|
+
anchor: result,
|
|
3610
|
+
candidates: samePath,
|
|
3611
|
+
maxGapLines: input.config.max_contiguous_gap_lines
|
|
3612
|
+
});
|
|
3613
|
+
const originalOmittedBefore = Math.max(0, result.start_line - envelope.start_line);
|
|
3614
|
+
const originalOmittedAfter = Math.max(0, envelope.end_line - result.end_line);
|
|
3615
|
+
const originalLooksTruncated = shouldAnnotateSnippetAsTruncated(result, originalOmittedBefore, originalOmittedAfter);
|
|
3616
|
+
if (!originalLooksTruncated) {
|
|
3617
|
+
return result;
|
|
3618
|
+
}
|
|
3619
|
+
const envelopeCandidates = samePath
|
|
3620
|
+
.filter((candidate) => candidate.end_line >= envelope.start_line && candidate.start_line <= envelope.end_line)
|
|
3621
|
+
.sort(compareSearchResultsByLineRange);
|
|
3622
|
+
let assembled = result;
|
|
3623
|
+
if (input.config.repair_enabled) {
|
|
3624
|
+
input.observability.metrics.increment("retrieval_snippet_repair_attempt_total", 1, {
|
|
3625
|
+
retrieval_profile_id: input.retrievalProfileId,
|
|
3626
|
+
language
|
|
3627
|
+
});
|
|
3628
|
+
const repairOutcome = repairSnippetFromEnvelope({
|
|
3629
|
+
anchor: result,
|
|
3630
|
+
envelope,
|
|
3631
|
+
samePathCandidates: samePath,
|
|
3632
|
+
config: input.config
|
|
3633
|
+
});
|
|
3634
|
+
if (repairOutcome.repaired) {
|
|
3635
|
+
assembled = repairOutcome.repaired;
|
|
3636
|
+
input.observability.metrics.increment("retrieval_snippet_repair_success_total", 1, {
|
|
3637
|
+
retrieval_profile_id: input.retrievalProfileId,
|
|
3638
|
+
language,
|
|
3639
|
+
clipped: repairOutcome.clipped ? "true" : "false"
|
|
3640
|
+
});
|
|
3641
|
+
input.observability.logger.info("snippet integrity repair decision", {
|
|
3642
|
+
retrieval_profile_id: input.retrievalProfileId,
|
|
3643
|
+
path: result.path,
|
|
3644
|
+
language,
|
|
3645
|
+
envelope_start_line: envelope.start_line,
|
|
3646
|
+
envelope_end_line: envelope.end_line,
|
|
3647
|
+
envelope_span_lines: envelope.end_line - envelope.start_line + 1,
|
|
3648
|
+
status: "repaired",
|
|
3649
|
+
clipped: repairOutcome.clipped
|
|
3650
|
+
});
|
|
3651
|
+
}
|
|
3652
|
+
else {
|
|
3653
|
+
input.observability.logger.info("snippet integrity repair decision", {
|
|
3654
|
+
retrieval_profile_id: input.retrievalProfileId,
|
|
3655
|
+
path: result.path,
|
|
3656
|
+
language,
|
|
3657
|
+
envelope_start_line: envelope.start_line,
|
|
3658
|
+
envelope_end_line: envelope.end_line,
|
|
3659
|
+
envelope_span_lines: envelope.end_line - envelope.start_line + 1,
|
|
3660
|
+
status: "repair_skipped",
|
|
3661
|
+
reason: repairOutcome.reason ?? "unknown"
|
|
3662
|
+
});
|
|
3663
|
+
}
|
|
3664
|
+
}
|
|
3665
|
+
const omittedBefore = Math.max(0, assembled.start_line - envelope.start_line);
|
|
3666
|
+
const omittedAfter = Math.max(0, envelope.end_line - assembled.end_line);
|
|
3667
|
+
if (!shouldAnnotateSnippetAsTruncated(assembled, omittedBefore, omittedAfter)) {
|
|
3668
|
+
return assembled;
|
|
3669
|
+
}
|
|
3670
|
+
let symbolName = detectSnippetSymbolName(assembled.snippet);
|
|
3671
|
+
if (!symbolName) {
|
|
3672
|
+
for (const candidate of envelopeCandidates) {
|
|
3673
|
+
symbolName = detectSnippetSymbolName(candidate.snippet);
|
|
3674
|
+
if (symbolName) {
|
|
3675
|
+
break;
|
|
3676
|
+
}
|
|
3677
|
+
}
|
|
3678
|
+
}
|
|
3679
|
+
const marker = buildSnippetTruncationMarker({
|
|
3680
|
+
result: assembled,
|
|
3681
|
+
symbolName,
|
|
3682
|
+
envelope_start_line: envelope.start_line,
|
|
3683
|
+
envelope_end_line: envelope.end_line,
|
|
3684
|
+
marker_template_version: input.config.marker_template_version
|
|
3685
|
+
});
|
|
3686
|
+
input.observability.metrics.increment("retrieval_snippet_repair_fallback_marker_total", 1, {
|
|
3687
|
+
retrieval_profile_id: input.retrievalProfileId,
|
|
3688
|
+
language
|
|
3689
|
+
});
|
|
3690
|
+
input.observability.metrics.increment("retrieval_snippet_truncation_marker_total", 1, {
|
|
3691
|
+
retrieval_profile_id: input.retrievalProfileId,
|
|
3692
|
+
language,
|
|
3693
|
+
symbol_detected: symbolName ? "true" : "false",
|
|
3694
|
+
marker_template_version: input.config.marker_template_version
|
|
3695
|
+
});
|
|
3696
|
+
input.observability.metrics.observe("retrieval_snippet_omitted_after_lines", omittedAfter, {
|
|
3697
|
+
retrieval_profile_id: input.retrievalProfileId,
|
|
3698
|
+
language
|
|
3699
|
+
});
|
|
3700
|
+
const baseSnippet = assembled.snippet.trimEnd();
|
|
3701
|
+
return {
|
|
3702
|
+
...assembled,
|
|
3703
|
+
snippet: baseSnippet.length > 0 ? `${baseSnippet}\n${marker}` : marker
|
|
3704
|
+
};
|
|
3705
|
+
});
|
|
3706
|
+
}
|
|
3707
|
+
function packSearchResultsWithContext(input) {
|
|
3708
|
+
if (!input.config.enabled || input.selected.length === 0) {
|
|
3709
|
+
return [...input.selected];
|
|
3710
|
+
}
|
|
3711
|
+
const sourceByPath = new Map();
|
|
3712
|
+
for (const candidate of input.sourceCandidates) {
|
|
3713
|
+
const rows = sourceByPath.get(candidate.path);
|
|
3714
|
+
if (rows) {
|
|
3715
|
+
rows.push(candidate);
|
|
3716
|
+
}
|
|
3717
|
+
else {
|
|
3718
|
+
sourceByPath.set(candidate.path, [candidate]);
|
|
3719
|
+
}
|
|
3720
|
+
}
|
|
3721
|
+
return input.selected.map((anchor) => {
|
|
3722
|
+
const samePath = sourceByPath.get(anchor.path) ?? [anchor];
|
|
3723
|
+
if (samePath.length <= 1 || input.config.max_spans_per_result <= 1) {
|
|
3724
|
+
return anchor;
|
|
3725
|
+
}
|
|
3726
|
+
const anchorRange = { start_line: anchor.start_line, end_line: anchor.end_line };
|
|
3727
|
+
const candidates = samePath
|
|
3728
|
+
.filter((candidate) => !(candidate.start_line === anchor.start_line && candidate.end_line === anchor.end_line) &&
|
|
3729
|
+
!isHeavilyOverlappingLineRange(candidate, [anchorRange]) &&
|
|
3730
|
+
lineRangeGap(anchorRange, candidate) <= input.config.max_gap_lines)
|
|
3731
|
+
.sort((a, b) => {
|
|
3732
|
+
const relevanceDiff = compareSearchResults(a, b);
|
|
3733
|
+
if (relevanceDiff !== 0) {
|
|
3734
|
+
return relevanceDiff;
|
|
3735
|
+
}
|
|
3736
|
+
return lineRangeGap(anchorRange, a) - lineRangeGap(anchorRange, b);
|
|
3737
|
+
});
|
|
3738
|
+
const spans = [{ ...anchorRange }];
|
|
3739
|
+
for (const candidate of candidates) {
|
|
3740
|
+
if (spans.length >= input.config.max_spans_per_result) {
|
|
3741
|
+
break;
|
|
3742
|
+
}
|
|
3743
|
+
const nextSpan = { start_line: candidate.start_line, end_line: candidate.end_line };
|
|
3744
|
+
const nextEnvelope = mergeLineSpans([...spans, nextSpan]);
|
|
3745
|
+
if (nextEnvelope.some((span, idx) => idx > 0 && span.start_line - (nextEnvelope[idx - 1]?.end_line ?? span.start_line) - 1 > input.config.max_gap_lines)) {
|
|
3746
|
+
continue;
|
|
3747
|
+
}
|
|
3748
|
+
spans.push(nextSpan);
|
|
3749
|
+
}
|
|
3750
|
+
const mergedSpans = mergeLineSpans(spans);
|
|
3751
|
+
if (mergedSpans.length <= 1) {
|
|
3752
|
+
return anchor;
|
|
3753
|
+
}
|
|
3754
|
+
const lineMap = buildPreferredLineMap([anchor, ...samePath]);
|
|
3755
|
+
const renderedLines = [];
|
|
3756
|
+
let contentLineCount = 0;
|
|
3757
|
+
let elisionCount = 0;
|
|
3758
|
+
for (let index = 0; index < mergedSpans.length; index += 1) {
|
|
3759
|
+
const span = mergedSpans[index];
|
|
3760
|
+
if (!span) {
|
|
3761
|
+
continue;
|
|
3762
|
+
}
|
|
3763
|
+
if (index > 0) {
|
|
3764
|
+
const previous = mergedSpans[index - 1];
|
|
3765
|
+
if (previous && span.start_line - previous.end_line > 0) {
|
|
3766
|
+
renderedLines.push("...");
|
|
3767
|
+
elisionCount += 1;
|
|
3768
|
+
}
|
|
3769
|
+
}
|
|
3770
|
+
for (let line = span.start_line; line <= span.end_line; line += 1) {
|
|
3771
|
+
renderedLines.push(lineMap.get(line) ?? "");
|
|
3772
|
+
contentLineCount += 1;
|
|
3773
|
+
}
|
|
3774
|
+
}
|
|
3775
|
+
if (renderedLines.length === 0) {
|
|
3776
|
+
return anchor;
|
|
3777
|
+
}
|
|
3778
|
+
const elisionDensity = elisionCount / Math.max(1, contentLineCount + elisionCount);
|
|
3779
|
+
if (elisionDensity > 0.25) {
|
|
3780
|
+
return anchor;
|
|
3781
|
+
}
|
|
3782
|
+
const packedSnippet = clipSnippetToMaxChars(renderedLines.join("\n"), input.config.max_snippet_chars);
|
|
3783
|
+
if (packedSnippet.length === 0) {
|
|
3784
|
+
return anchor;
|
|
3785
|
+
}
|
|
3786
|
+
const packedStart = mergedSpans[0]?.start_line ?? anchor.start_line;
|
|
3787
|
+
const packedEnd = mergedSpans[mergedSpans.length - 1]?.end_line ?? anchor.end_line;
|
|
3788
|
+
return {
|
|
3789
|
+
...anchor,
|
|
3790
|
+
start_line: packedStart,
|
|
3791
|
+
end_line: packedEnd,
|
|
3792
|
+
snippet: packedSnippet,
|
|
3793
|
+
reason: `${anchor.reason} + contextual spans`
|
|
3794
|
+
};
|
|
3795
|
+
});
|
|
3796
|
+
}
|
|
3797
|
+
function mergeOverlappingCandidates(candidates, config) {
|
|
3798
|
+
if (!config.merge_overlapping_chunks_enabled || candidates.length <= 1) {
|
|
3799
|
+
return [...candidates];
|
|
1618
3800
|
}
|
|
1619
|
-
const
|
|
1620
|
-
|
|
1621
|
-
|
|
3801
|
+
const byPath = new Map();
|
|
3802
|
+
for (const candidate of candidates) {
|
|
3803
|
+
const group = byPath.get(candidate.path);
|
|
3804
|
+
if (group) {
|
|
3805
|
+
group.push(candidate);
|
|
3806
|
+
}
|
|
3807
|
+
else {
|
|
3808
|
+
byPath.set(candidate.path, [candidate]);
|
|
3809
|
+
}
|
|
1622
3810
|
}
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
3811
|
+
const merged = [];
|
|
3812
|
+
for (const group of byPath.values()) {
|
|
3813
|
+
const ordered = [...group].sort(compareSearchResultsByLineRange);
|
|
3814
|
+
let cluster = [];
|
|
3815
|
+
let clusterStart = 0;
|
|
3816
|
+
let clusterEnd = 0;
|
|
3817
|
+
const flush = () => {
|
|
3818
|
+
if (cluster.length === 0) {
|
|
3819
|
+
return;
|
|
3820
|
+
}
|
|
3821
|
+
merged.push(mergeCandidateCluster(cluster));
|
|
3822
|
+
cluster = [];
|
|
3823
|
+
};
|
|
3824
|
+
for (const candidate of ordered) {
|
|
3825
|
+
if (cluster.length === 0) {
|
|
3826
|
+
cluster = [candidate];
|
|
3827
|
+
clusterStart = candidate.start_line;
|
|
3828
|
+
clusterEnd = candidate.end_line;
|
|
3829
|
+
continue;
|
|
3830
|
+
}
|
|
3831
|
+
const nextStart = Math.min(clusterStart, candidate.start_line);
|
|
3832
|
+
const nextEnd = Math.max(clusterEnd, candidate.end_line);
|
|
3833
|
+
const nextSpan = nextEnd - nextStart + 1;
|
|
3834
|
+
const gapLines = Math.max(0, candidate.start_line - clusterEnd - 1);
|
|
3835
|
+
const canMerge = gapLines <= config.merge_gap_lines && nextSpan <= config.merge_max_span_lines;
|
|
3836
|
+
if (!canMerge) {
|
|
3837
|
+
flush();
|
|
3838
|
+
cluster = [candidate];
|
|
3839
|
+
clusterStart = candidate.start_line;
|
|
3840
|
+
clusterEnd = candidate.end_line;
|
|
3841
|
+
continue;
|
|
3842
|
+
}
|
|
3843
|
+
cluster.push(candidate);
|
|
3844
|
+
clusterStart = nextStart;
|
|
3845
|
+
clusterEnd = nextEnd;
|
|
1630
3846
|
}
|
|
3847
|
+
flush();
|
|
1631
3848
|
}
|
|
1632
|
-
|
|
1633
|
-
return tolerated.length > 0 ? tolerated : results;
|
|
3849
|
+
return merged.sort(compareSearchResults);
|
|
1634
3850
|
}
|
|
1635
|
-
function
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
3851
|
+
function applySmartCutoffCandidates(candidates, config) {
|
|
3852
|
+
if (!config.smart_cutoff_enabled || candidates.length === 0) {
|
|
3853
|
+
return [...candidates];
|
|
3854
|
+
}
|
|
3855
|
+
const ordered = [...candidates].sort(compareSearchResults);
|
|
3856
|
+
const minK = Math.max(1, config.smart_cutoff_min_k);
|
|
3857
|
+
const maxK = Math.max(minK, config.smart_cutoff_max_k);
|
|
3858
|
+
const topScore = ordered[0]?.score ?? Number.NEGATIVE_INFINITY;
|
|
3859
|
+
const kept = [];
|
|
3860
|
+
for (let index = 0; index < ordered.length; index += 1) {
|
|
3861
|
+
const candidate = ordered[index];
|
|
3862
|
+
if (!candidate) {
|
|
3863
|
+
continue;
|
|
3864
|
+
}
|
|
3865
|
+
if (kept.length >= maxK) {
|
|
3866
|
+
break;
|
|
3867
|
+
}
|
|
3868
|
+
if (kept.length < minK) {
|
|
3869
|
+
kept.push(candidate);
|
|
3870
|
+
continue;
|
|
3871
|
+
}
|
|
3872
|
+
if (candidate.score < config.smart_cutoff_min_score) {
|
|
3873
|
+
break;
|
|
3874
|
+
}
|
|
3875
|
+
if (candidate.score < topScore * config.smart_cutoff_top_ratio) {
|
|
3876
|
+
break;
|
|
3877
|
+
}
|
|
3878
|
+
const previous = ordered[index - 1];
|
|
3879
|
+
if (previous && previous.score - candidate.score > config.smart_cutoff_delta_abs) {
|
|
3880
|
+
break;
|
|
3881
|
+
}
|
|
3882
|
+
kept.push(candidate);
|
|
1645
3883
|
}
|
|
1646
|
-
return
|
|
3884
|
+
return kept;
|
|
3885
|
+
}
|
|
3886
|
+
export function __applySmartCutoffCandidatesForTests(input) {
|
|
3887
|
+
return applySmartCutoffCandidates(input.candidates, input.config);
|
|
1647
3888
|
}
|
|
1648
3889
|
function dedupeEnhancerCandidatesByPath(results) {
|
|
1649
3890
|
const byPath = new Map();
|
|
@@ -1701,7 +3942,7 @@ function hasStrongEnhancerAnchorMatch(input) {
|
|
|
1701
3942
|
const topScore = top[0]?.score ?? 0;
|
|
1702
3943
|
const runnerUpScore = top[1]?.score ?? Number.NEGATIVE_INFINITY;
|
|
1703
3944
|
const strongScoreMargin = top.length === 1 || topScore - runnerUpScore >= 0.08;
|
|
1704
|
-
const hasTopExactSymbolMatch = top.some((result) => result.reason
|
|
3945
|
+
const hasTopExactSymbolMatch = top.some((result) => isExactLiteralReason(result.reason));
|
|
1705
3946
|
if (hasTopExactSymbolMatch && strongScoreMargin && topScore >= 0.55) {
|
|
1706
3947
|
return true;
|
|
1707
3948
|
}
|
|
@@ -1792,7 +4033,7 @@ function evaluateEnhancerConfidence(input) {
|
|
|
1792
4033
|
if (diversityStrength < confidenceThreshold) {
|
|
1793
4034
|
failedSignals.push("path_diversity");
|
|
1794
4035
|
}
|
|
1795
|
-
const strongSymbolOrPathSignal = top.some((result) => result.reason
|
|
4036
|
+
const strongSymbolOrPathSignal = top.some((result) => isExactLiteralReason(result.reason)) && topOverlap >= 0.16;
|
|
1796
4037
|
const lowConfidence = !strongSymbolOrPathSignal && confidenceScore + 0.01 < confidenceThreshold;
|
|
1797
4038
|
return {
|
|
1798
4039
|
score_spread: scoreSpread,
|
|
@@ -1810,7 +4051,7 @@ function rankEnhancerResultsForConfidence(input) {
|
|
|
1810
4051
|
const anchorScore = (result) => {
|
|
1811
4052
|
const normalizedPath = normalizePath(result.path).toLowerCase();
|
|
1812
4053
|
const normalizedSnippet = result.snippet.toLowerCase();
|
|
1813
|
-
let score = result.reason
|
|
4054
|
+
let score = isExactLiteralReason(result.reason) ? 2 : 0;
|
|
1814
4055
|
for (const anchor of anchors) {
|
|
1815
4056
|
if (normalizedPath.includes(anchor)) {
|
|
1816
4057
|
score += 2;
|
|
@@ -1869,6 +4110,7 @@ async function runWithTimeout(input) {
|
|
|
1869
4110
|
return;
|
|
1870
4111
|
}
|
|
1871
4112
|
settled = true;
|
|
4113
|
+
input.on_timeout?.();
|
|
1872
4114
|
reject(new Error(`timeout_after_${input.timeout_ms}ms`));
|
|
1873
4115
|
}, input.timeout_ms);
|
|
1874
4116
|
Promise.resolve()
|
|
@@ -1891,50 +4133,67 @@ async function runWithTimeout(input) {
|
|
|
1891
4133
|
});
|
|
1892
4134
|
});
|
|
1893
4135
|
}
|
|
4136
|
+
async function runWithInactivityTimeout(input) {
|
|
4137
|
+
return await new Promise((resolve, reject) => {
|
|
4138
|
+
let settled = false;
|
|
4139
|
+
const abortController = new AbortController();
|
|
4140
|
+
let timer;
|
|
4141
|
+
const onTimeout = () => {
|
|
4142
|
+
if (settled) {
|
|
4143
|
+
return;
|
|
4144
|
+
}
|
|
4145
|
+
settled = true;
|
|
4146
|
+
abortController.abort();
|
|
4147
|
+
reject(new Error(`timeout_after_${input.timeout_ms}ms`));
|
|
4148
|
+
};
|
|
4149
|
+
const touch = () => {
|
|
4150
|
+
if (settled) {
|
|
4151
|
+
return;
|
|
4152
|
+
}
|
|
4153
|
+
if (timer) {
|
|
4154
|
+
clearTimeout(timer);
|
|
4155
|
+
}
|
|
4156
|
+
timer = setTimeout(onTimeout, input.timeout_ms);
|
|
4157
|
+
};
|
|
4158
|
+
touch();
|
|
4159
|
+
Promise.resolve()
|
|
4160
|
+
.then(() => input.fn({
|
|
4161
|
+
touch,
|
|
4162
|
+
signal: abortController.signal
|
|
4163
|
+
}))
|
|
4164
|
+
.then((value) => {
|
|
4165
|
+
if (settled) {
|
|
4166
|
+
return;
|
|
4167
|
+
}
|
|
4168
|
+
settled = true;
|
|
4169
|
+
if (timer) {
|
|
4170
|
+
clearTimeout(timer);
|
|
4171
|
+
}
|
|
4172
|
+
resolve(value);
|
|
4173
|
+
})
|
|
4174
|
+
.catch((error) => {
|
|
4175
|
+
if (settled) {
|
|
4176
|
+
return;
|
|
4177
|
+
}
|
|
4178
|
+
settled = true;
|
|
4179
|
+
if (timer) {
|
|
4180
|
+
clearTimeout(timer);
|
|
4181
|
+
}
|
|
4182
|
+
reject(error);
|
|
4183
|
+
});
|
|
4184
|
+
});
|
|
4185
|
+
}
|
|
1894
4186
|
function deterministicEnhancerFallbackRanking(input) {
|
|
1895
4187
|
const preferred = input.results.filter((result) => !isRiskyEnhancerPath(result.path, input.intent) && !shouldAvoidPathFromNegation(result.path, input.negative_preferences));
|
|
1896
4188
|
const tolerated = input.results.filter((result) => !preferred.includes(result) && !shouldAvoidPathFromNegation(result.path, input.negative_preferences));
|
|
1897
4189
|
const avoided = input.results.filter((result) => !preferred.includes(result) && !tolerated.includes(result));
|
|
1898
4190
|
return [...preferred, ...tolerated, ...avoided];
|
|
1899
4191
|
}
|
|
1900
|
-
function
|
|
1901
|
-
if (input.kind === "symbol") {
|
|
1902
|
-
if (input.language === "es") {
|
|
1903
|
-
return input.symbol
|
|
1904
|
-
? `¿Puedes confirmar si el cambio debe centrarse en el símbolo "${input.symbol}"?`
|
|
1905
|
-
: "¿Qué función, clase o archivo exacto debe modificarse primero?";
|
|
1906
|
-
}
|
|
1907
|
-
if (input.language === "zh") {
|
|
1908
|
-
return input.symbol
|
|
1909
|
-
? `请确认这次改动是否应优先围绕符号“${input.symbol}”展开?`
|
|
1910
|
-
: "请明确首先要修改的函数、类或文件路径。";
|
|
1911
|
-
}
|
|
1912
|
-
return input.symbol
|
|
1913
|
-
? `Can you confirm whether "${input.symbol}" is the primary symbol to change?`
|
|
1914
|
-
: "Which exact function, class, or file should be edited first?";
|
|
1915
|
-
}
|
|
1916
|
-
if (input.kind === "source_priority") {
|
|
1917
|
-
if (input.language === "es") {
|
|
1918
|
-
return "¿Debemos priorizar archivos de implementación en src/lib y dejar docs/tests/examples fuera de alcance?";
|
|
1919
|
-
}
|
|
1920
|
-
if (input.language === "zh") {
|
|
1921
|
-
return "是否应优先修改 src/lib 下的实现代码,并排除 docs/tests/examples?";
|
|
1922
|
-
}
|
|
1923
|
-
return "Should we prioritize runtime implementation files (src/lib) and exclude docs/tests/examples from scope?";
|
|
1924
|
-
}
|
|
1925
|
-
if (input.language === "es") {
|
|
1926
|
-
return "¿Cuál es el alcance mínimo y el comportamiento que no debe cambiar?";
|
|
1927
|
-
}
|
|
1928
|
-
if (input.language === "zh") {
|
|
1929
|
-
return "这次改动的最小范围是什么?哪些行为必须保持不变?";
|
|
1930
|
-
}
|
|
1931
|
-
return "What is the minimal scope, and which behavior must remain unchanged?";
|
|
1932
|
-
}
|
|
1933
|
-
function trimToContextBudget(results) {
|
|
4192
|
+
function trimToContextBudget(results, budgetTokenizerMode) {
|
|
1934
4193
|
let total = 0;
|
|
1935
4194
|
const out = [];
|
|
1936
4195
|
for (const result of results) {
|
|
1937
|
-
total +=
|
|
4196
|
+
total += chunkBudgetTokenize(result.snippet, budgetTokenizerMode).length;
|
|
1938
4197
|
if (total > MAX_CONTEXT_BUDGET_TOKENS) {
|
|
1939
4198
|
break;
|
|
1940
4199
|
}
|
|
@@ -1950,89 +4209,190 @@ function formatEnhancedPrompt(input) {
|
|
|
1950
4209
|
};
|
|
1951
4210
|
const likelyFiles = input.refs.length > 0 ? input.refs.map((r) => `- ${r.path}:${r.start_line}`).join("\n") : emptyRefsByLanguage[input.language];
|
|
1952
4211
|
if (input.language === "zh") {
|
|
4212
|
+
if (input.style === "lean") {
|
|
4213
|
+
return [
|
|
4214
|
+
"目标",
|
|
4215
|
+
input.original_prompt,
|
|
4216
|
+
"",
|
|
4217
|
+
"约束",
|
|
4218
|
+
"- 保持现有行为与合约兼容。",
|
|
4219
|
+
"- 优先最小且安全的改动。",
|
|
4220
|
+
"",
|
|
4221
|
+
"行动步骤",
|
|
4222
|
+
"- 先确认当前行为与目标范围。",
|
|
4223
|
+
"- 在必要位置完成最小实现并补充回归测试。",
|
|
4224
|
+
"",
|
|
4225
|
+
"验证",
|
|
4226
|
+
"- 运行相关测试并确认无回归。"
|
|
4227
|
+
].join("\n");
|
|
4228
|
+
}
|
|
4229
|
+
if (input.style === "deep") {
|
|
4230
|
+
return [
|
|
4231
|
+
"目标",
|
|
4232
|
+
input.original_prompt,
|
|
4233
|
+
"",
|
|
4234
|
+
"范围与约束",
|
|
4235
|
+
"- 保持现有行为与 API/合约语义稳定。",
|
|
4236
|
+
"- 仅在必要边界内调整实现,避免扩散改动。",
|
|
4237
|
+
"- 发现风险路径时优先失败安全(deny-by-default)。",
|
|
4238
|
+
"",
|
|
4239
|
+
"代码锚点",
|
|
4240
|
+
likelyFiles,
|
|
4241
|
+
"",
|
|
4242
|
+
"实施步骤",
|
|
4243
|
+
"- 基线确认:先验证当前行为与关键路径。",
|
|
4244
|
+
"- 变更实现:对关键分支做最小、安全、可回退的改动。",
|
|
4245
|
+
"- 回归测试:覆盖正向、跨边界、异常与空输入场景。",
|
|
4246
|
+
"",
|
|
4247
|
+
"边界情况",
|
|
4248
|
+
"- 缺失上下文、无索引或空结果时,保持行为可解释且可回退。",
|
|
4249
|
+
"- 异步/并发路径中避免上下文泄漏与跨租户访问。",
|
|
4250
|
+
"",
|
|
4251
|
+
"验证",
|
|
4252
|
+
"- 运行 typecheck 与目标测试集;确认关键路径稳定无回归。"
|
|
4253
|
+
].join("\n");
|
|
4254
|
+
}
|
|
1953
4255
|
return [
|
|
1954
4256
|
"目标",
|
|
1955
4257
|
input.original_prompt,
|
|
1956
4258
|
"",
|
|
1957
|
-
"当前状态",
|
|
1958
|
-
`- 识别意图: ${input.intent}`,
|
|
1959
|
-
"",
|
|
1960
4259
|
"约束",
|
|
1961
4260
|
"- 保持 v1 合约兼容和严格校验。",
|
|
1962
4261
|
"",
|
|
1963
|
-
"
|
|
4262
|
+
"代码锚点",
|
|
1964
4263
|
likelyFiles,
|
|
1965
4264
|
"",
|
|
1966
4265
|
"实现清单",
|
|
1967
4266
|
"- 在改动前确认请求/响应合约。",
|
|
1968
4267
|
"- 最小化改动并保持 tenant/workspace 隔离。",
|
|
1969
4268
|
"",
|
|
1970
|
-
"边界情况",
|
|
1971
|
-
"- Workspace 没有可用索引。",
|
|
1972
|
-
"- 搜索过滤后结果为空。",
|
|
1973
|
-
"",
|
|
1974
4269
|
"验证与测试",
|
|
1975
4270
|
"- 运行 typecheck 和合约/工具测试。",
|
|
1976
|
-
"",
|
|
1977
|
-
"完成定义",
|
|
1978
|
-
"- 测试通过且行为符合 v1 规范。"
|
|
1979
4271
|
].join("\n");
|
|
1980
4272
|
}
|
|
1981
4273
|
if (input.language === "es") {
|
|
4274
|
+
if (input.style === "lean") {
|
|
4275
|
+
return [
|
|
4276
|
+
"Objetivo",
|
|
4277
|
+
input.original_prompt,
|
|
4278
|
+
"",
|
|
4279
|
+
"Restricciones",
|
|
4280
|
+
"- Mantener compatibilidad de comportamiento y contratos.",
|
|
4281
|
+
"- Priorizar cambios mínimos y seguros.",
|
|
4282
|
+
"",
|
|
4283
|
+
"Pasos",
|
|
4284
|
+
"- Confirmar alcance y comportamiento actual antes de editar.",
|
|
4285
|
+
"- Implementar el cambio mínimo necesario y añadir regresiones.",
|
|
4286
|
+
"",
|
|
4287
|
+
"Validación",
|
|
4288
|
+
"- Ejecutar pruebas relevantes y confirmar que no hay regresiones."
|
|
4289
|
+
].join("\n");
|
|
4290
|
+
}
|
|
4291
|
+
if (input.style === "deep") {
|
|
4292
|
+
return [
|
|
4293
|
+
"Objetivo",
|
|
4294
|
+
input.original_prompt,
|
|
4295
|
+
"",
|
|
4296
|
+
"Alcance y restricciones",
|
|
4297
|
+
"- Preservar comportamiento existente y contratos/API vigentes.",
|
|
4298
|
+
"- Limitar cambios al alcance mínimo necesario.",
|
|
4299
|
+
"- Aplicar defaults de seguridad (deny-by-default) cuando aplique.",
|
|
4300
|
+
"",
|
|
4301
|
+
"Anclas del código",
|
|
4302
|
+
likelyFiles,
|
|
4303
|
+
"",
|
|
4304
|
+
"Plan de implementación",
|
|
4305
|
+
"- Establecer línea base del comportamiento actual.",
|
|
4306
|
+
"- Aplicar cambios mínimos y reversibles en rutas críticas.",
|
|
4307
|
+
"- Añadir pruebas de regresión para casos positivos, negativos y límites.",
|
|
4308
|
+
"",
|
|
4309
|
+
"Casos límite",
|
|
4310
|
+
"- Contexto faltante o resultados vacíos no deben romper el flujo.",
|
|
4311
|
+
"- Evitar fuga de contexto entre tenants/workspaces.",
|
|
4312
|
+
"",
|
|
4313
|
+
"Validación",
|
|
4314
|
+
"- Ejecutar typecheck y pruebas objetivo; confirmar estabilidad."
|
|
4315
|
+
].join("\n");
|
|
4316
|
+
}
|
|
1982
4317
|
return [
|
|
1983
4318
|
"Objetivo",
|
|
1984
4319
|
input.original_prompt,
|
|
1985
4320
|
"",
|
|
1986
|
-
"Estado actual",
|
|
1987
|
-
`- Intención clasificada: ${input.intent}`,
|
|
1988
|
-
"",
|
|
1989
4321
|
"Restricciones",
|
|
1990
4322
|
"- Mantener compatibilidad con contratos v1 y validación estricta.",
|
|
1991
4323
|
"",
|
|
1992
|
-
"
|
|
4324
|
+
"Anclas del código",
|
|
1993
4325
|
likelyFiles,
|
|
1994
4326
|
"",
|
|
1995
4327
|
"Checklist de implementación",
|
|
1996
4328
|
"- Confirmar entradas/salidas del contrato antes de modificar lógica.",
|
|
1997
4329
|
"- Aplicar cambios mínimos y mantener aislamiento por tenant/workspace.",
|
|
1998
4330
|
"",
|
|
1999
|
-
"Casos límite",
|
|
2000
|
-
"- Workspace sin índice listo.",
|
|
2001
|
-
"- Filtros de búsqueda que no devuelven resultados.",
|
|
2002
|
-
"",
|
|
2003
4331
|
"Validación y pruebas",
|
|
2004
|
-
"- Ejecutar typecheck y pruebas de contratos/herramientas."
|
|
4332
|
+
"- Ejecutar typecheck y pruebas de contratos/herramientas."
|
|
4333
|
+
].join("\n");
|
|
4334
|
+
}
|
|
4335
|
+
if (input.style === "lean") {
|
|
4336
|
+
const anchors = input.refs.length > 0 ? `- Anchors: ${input.refs.slice(0, 2).map((ref) => `${ref.path}:${ref.start_line}`).join(", ")}` : "";
|
|
4337
|
+
return [
|
|
4338
|
+
"Goal",
|
|
4339
|
+
input.original_prompt,
|
|
4340
|
+
"",
|
|
4341
|
+
"Constraints",
|
|
4342
|
+
"- Preserve existing behavior and contract compatibility.",
|
|
4343
|
+
"- Keep changes minimal and safe.",
|
|
4344
|
+
...(anchors ? ["", anchors] : []),
|
|
4345
|
+
"",
|
|
4346
|
+
"Action steps",
|
|
4347
|
+
"- Confirm current behavior and target scope.",
|
|
4348
|
+
"- Implement the smallest safe change and add regression coverage.",
|
|
4349
|
+
"",
|
|
4350
|
+
"Validation",
|
|
4351
|
+
"- Run relevant tests and confirm no regressions."
|
|
4352
|
+
].join("\n");
|
|
4353
|
+
}
|
|
4354
|
+
if (input.style === "deep") {
|
|
4355
|
+
return [
|
|
4356
|
+
"Goal",
|
|
4357
|
+
input.original_prompt,
|
|
2005
4358
|
"",
|
|
2006
|
-
"
|
|
2007
|
-
"-
|
|
4359
|
+
"Scope and constraints",
|
|
4360
|
+
"- Preserve current behavior and API/contract semantics.",
|
|
4361
|
+
"- Limit changes to the required scope and keep them reversible.",
|
|
4362
|
+
"- Prefer fail-secure defaults where policy boundaries are involved.",
|
|
4363
|
+
"",
|
|
4364
|
+
"Codebase anchors",
|
|
4365
|
+
likelyFiles,
|
|
4366
|
+
"",
|
|
4367
|
+
"Implementation plan",
|
|
4368
|
+
"- Establish baseline behavior and invariants before edits.",
|
|
4369
|
+
"- Apply minimal, safe changes on critical paths only.",
|
|
4370
|
+
"- Add regression coverage for positive, negative, and boundary scenarios.",
|
|
4371
|
+
"",
|
|
4372
|
+
"Edge cases",
|
|
4373
|
+
"- Missing context, empty retrieval results, and async boundary leakage.",
|
|
4374
|
+
"- Cross-tenant/workspace access paths and authorization bypass attempts.",
|
|
4375
|
+
"",
|
|
4376
|
+
"Validation",
|
|
4377
|
+
"- Run typecheck and focused test suites; verify no behavioral regressions."
|
|
2008
4378
|
].join("\n");
|
|
2009
4379
|
}
|
|
2010
4380
|
return [
|
|
2011
4381
|
"Goal",
|
|
2012
4382
|
input.original_prompt,
|
|
2013
4383
|
"",
|
|
2014
|
-
"Current state",
|
|
2015
|
-
`- Classified intent: ${input.intent}`,
|
|
2016
|
-
"",
|
|
2017
4384
|
"Constraints",
|
|
2018
4385
|
"- Keep v1 contract compatibility and strict schema validation.",
|
|
2019
4386
|
"",
|
|
2020
|
-
"
|
|
4387
|
+
"Codebase anchors",
|
|
2021
4388
|
likelyFiles,
|
|
2022
4389
|
"",
|
|
2023
|
-
"Implementation
|
|
4390
|
+
"Implementation plan",
|
|
2024
4391
|
"- Confirm request/response contract assumptions before code edits.",
|
|
2025
4392
|
"- Apply smallest safe changes while preserving tenant/workspace isolation.",
|
|
2026
4393
|
"",
|
|
2027
|
-
"Edge cases",
|
|
2028
|
-
"- Workspace has no ready index.",
|
|
2029
|
-
"- Search filters produce empty result sets.",
|
|
2030
|
-
"",
|
|
2031
4394
|
"Validation and tests",
|
|
2032
|
-
"- Run typecheck and contract/tool tests."
|
|
2033
|
-
"",
|
|
2034
|
-
"Definition of done",
|
|
2035
|
-
"- Tests pass and behavior matches the v1 spec."
|
|
4395
|
+
"- Run typecheck and contract/tool tests."
|
|
2036
4396
|
].join("\n");
|
|
2037
4397
|
}
|
|
2038
4398
|
function detectSecretMatches(content) {
|
|
@@ -2263,14 +4623,24 @@ export class RetrievalCore {
|
|
|
2263
4623
|
store;
|
|
2264
4624
|
cache;
|
|
2265
4625
|
cacheTtlSeconds;
|
|
4626
|
+
internalCandidateDepth;
|
|
2266
4627
|
embeddingProvider;
|
|
2267
4628
|
embeddingDescriptor;
|
|
4629
|
+
rerankerProvider;
|
|
4630
|
+
rerankerDescriptor;
|
|
4631
|
+
rerankerTopN;
|
|
4632
|
+
rerankerCacheVariant;
|
|
2268
4633
|
observability;
|
|
2269
4634
|
scoringConfig;
|
|
2270
4635
|
scoringProfileId;
|
|
2271
4636
|
scoringConfigChecksum;
|
|
4637
|
+
enhancerProvider;
|
|
4638
|
+
enhancerProviderDescriptor;
|
|
2272
4639
|
enhancerConfig;
|
|
4640
|
+
enhancerGenerationConfig;
|
|
2273
4641
|
chunkingConfig;
|
|
4642
|
+
contextPackingConfig;
|
|
4643
|
+
snippetIntegrityConfig;
|
|
2274
4644
|
enhancerDecisionTraceEnabled;
|
|
2275
4645
|
cacheHits = 0;
|
|
2276
4646
|
cacheMisses = 0;
|
|
@@ -2278,15 +4648,34 @@ export class RetrievalCore {
|
|
|
2278
4648
|
this.store = store;
|
|
2279
4649
|
this.cache = cache;
|
|
2280
4650
|
this.cacheTtlSeconds = options?.cacheTtlSeconds ?? 60;
|
|
4651
|
+
this.internalCandidateDepth = clampInternalCandidateDepth(options?.internalCandidateDepth);
|
|
2281
4652
|
this.embeddingProvider = options?.embeddingProvider ?? new DeterministicEmbeddingProvider();
|
|
2282
4653
|
this.embeddingDescriptor = normalizeEmbeddingDescriptor(options?.embeddingDescriptor ?? resolveEmbeddingDescriptor(this.embeddingProvider));
|
|
4654
|
+
this.rerankerProvider = options?.rerankerProvider;
|
|
4655
|
+
this.rerankerTopN = options?.rerankerTopN ?? DEFAULT_SEARCH_RERANKER_TOP_N;
|
|
4656
|
+
if (!Number.isInteger(this.rerankerTopN) || this.rerankerTopN <= 0) {
|
|
4657
|
+
throw new Error("invalid retrieval reranker config: rerankerTopN must be a positive integer");
|
|
4658
|
+
}
|
|
4659
|
+
this.rerankerDescriptor = this.rerankerProvider
|
|
4660
|
+
? normalizeRerankerDescriptor(resolveRerankerDescriptor(this.rerankerProvider))
|
|
4661
|
+
: undefined;
|
|
4662
|
+
this.rerankerCacheVariant = this.rerankerDescriptor
|
|
4663
|
+
? `provider:${this.rerankerDescriptor.provider}|model:${this.rerankerDescriptor.model ?? "unknown"}|top_n:${this.rerankerTopN}`
|
|
4664
|
+
: "provider:disabled";
|
|
2283
4665
|
this.observability = options?.observability ?? getObservability("retrieval-core");
|
|
2284
4666
|
const baseProfile = resolveRetrievalScoringProfile(options?.scoringProfile);
|
|
2285
4667
|
this.scoringConfig = mergeRetrievalScoringConfig(baseProfile.config, options?.scoringConfig);
|
|
2286
4668
|
this.scoringProfileId = options?.scoringProfileId ?? baseProfile.profile_id;
|
|
2287
4669
|
this.scoringConfigChecksum = scoringConfigChecksum(this.scoringConfig);
|
|
4670
|
+
this.enhancerProvider = options?.enhancerProvider;
|
|
4671
|
+
this.enhancerProviderDescriptor = this.enhancerProvider
|
|
4672
|
+
? normalizeEnhancerProviderDescriptor(resolveEnhancerProviderDescriptor(this.enhancerProvider))
|
|
4673
|
+
: undefined;
|
|
2288
4674
|
this.enhancerConfig = mergeRetrievalEnhancerConfig(DEFAULT_RETRIEVAL_ENHANCER_CONFIG, options?.enhancerConfig);
|
|
4675
|
+
this.enhancerGenerationConfig = mergeRetrievalEnhancerGenerationConfig(DEFAULT_RETRIEVAL_ENHANCER_GENERATION_CONFIG, options?.enhancerGenerationConfig);
|
|
2289
4676
|
this.chunkingConfig = mergeRetrievalChunkingConfig(DEFAULT_RETRIEVAL_CHUNKING_CONFIG, options?.chunkingConfig);
|
|
4677
|
+
this.contextPackingConfig = mergeRetrievalContextPackingConfig(DEFAULT_RETRIEVAL_CONTEXT_PACKING_CONFIG, options?.contextPackingConfig);
|
|
4678
|
+
this.snippetIntegrityConfig = mergeRetrievalSnippetIntegrityConfig(DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG, options?.snippetIntegrityConfig);
|
|
2290
4679
|
this.enhancerDecisionTraceEnabled = Boolean(options?.enhancerDecisionTraceEnabled);
|
|
2291
4680
|
}
|
|
2292
4681
|
async indexArtifact(artifact) {
|
|
@@ -2460,6 +4849,12 @@ export class RetrievalCore {
|
|
|
2460
4849
|
language: chunkLanguage,
|
|
2461
4850
|
reason: chunkBuild.fallback_reason ?? "none"
|
|
2462
4851
|
});
|
|
4852
|
+
if (chunkBuild.recursive_semantic_chunking_used) {
|
|
4853
|
+
this.observability.metrics.increment("index_recursive_semantic_chunking_used_total", 1, {
|
|
4854
|
+
tenant_id: artifact.tenant_id,
|
|
4855
|
+
language: chunkLanguage
|
|
4856
|
+
});
|
|
4857
|
+
}
|
|
2463
4858
|
if (chunkBuild.fallback_reason) {
|
|
2464
4859
|
this.observability.metrics.increment("index_chunking_fallback_total", 1, {
|
|
2465
4860
|
tenant_id: artifact.tenant_id,
|
|
@@ -2487,14 +4882,15 @@ export class RetrievalCore {
|
|
|
2487
4882
|
reason: chunkBuild.fallback_reason
|
|
2488
4883
|
});
|
|
2489
4884
|
}
|
|
2490
|
-
const
|
|
4885
|
+
const embeddingTexts = buildChunkEmbeddingTexts(chunks, this.chunkingConfig, this.embeddingDescriptor.provider);
|
|
4886
|
+
const estimatedEmbeddingTokens = embeddingTexts.reduce((sum, text) => sum + tokenize(text).length, 0);
|
|
2491
4887
|
this.observability.metrics.increment("index_embedding_tokens_total", estimatedEmbeddingTokens, {
|
|
2492
4888
|
tenant_id: artifact.tenant_id
|
|
2493
4889
|
});
|
|
2494
4890
|
const embeddings = chunks.length === 0
|
|
2495
4891
|
? []
|
|
2496
4892
|
: await this.embeddingProvider.embed({
|
|
2497
|
-
texts:
|
|
4893
|
+
texts: embeddingTexts,
|
|
2498
4894
|
purpose: "index"
|
|
2499
4895
|
});
|
|
2500
4896
|
if (embeddings.length !== chunks.length) {
|
|
@@ -2769,6 +5165,12 @@ export class RetrievalCore {
|
|
|
2769
5165
|
language: chunkLanguage,
|
|
2770
5166
|
reason: chunkBuild.fallback_reason ?? "none"
|
|
2771
5167
|
});
|
|
5168
|
+
if (chunkBuild.recursive_semantic_chunking_used) {
|
|
5169
|
+
this.observability.metrics.increment("index_recursive_semantic_chunking_used_total", 1, {
|
|
5170
|
+
tenant_id: artifact.tenant_id,
|
|
5171
|
+
language: chunkLanguage
|
|
5172
|
+
});
|
|
5173
|
+
}
|
|
2772
5174
|
if (chunkBuild.fallback_reason) {
|
|
2773
5175
|
this.observability.metrics.increment("index_chunking_fallback_total", 1, {
|
|
2774
5176
|
tenant_id: artifact.tenant_id,
|
|
@@ -2796,14 +5198,15 @@ export class RetrievalCore {
|
|
|
2796
5198
|
reason: chunkBuild.fallback_reason
|
|
2797
5199
|
});
|
|
2798
5200
|
}
|
|
2799
|
-
const
|
|
5201
|
+
const embeddingTexts = buildChunkEmbeddingTexts(chunks, this.chunkingConfig, this.embeddingDescriptor.provider);
|
|
5202
|
+
const estimatedEmbeddingTokens = embeddingTexts.reduce((sum, text) => sum + tokenize(text).length, 0);
|
|
2800
5203
|
this.observability.metrics.increment("index_embedding_tokens_total", estimatedEmbeddingTokens, {
|
|
2801
5204
|
tenant_id: artifact.tenant_id
|
|
2802
5205
|
});
|
|
2803
5206
|
const embeddings = chunks.length === 0
|
|
2804
5207
|
? []
|
|
2805
5208
|
: await this.embeddingProvider.embed({
|
|
2806
|
-
texts:
|
|
5209
|
+
texts: embeddingTexts,
|
|
2807
5210
|
purpose: "index"
|
|
2808
5211
|
});
|
|
2809
5212
|
if (embeddings.length !== chunks.length) {
|
|
@@ -2902,6 +5305,95 @@ export class RetrievalCore {
|
|
|
2902
5305
|
status: existing.status
|
|
2903
5306
|
};
|
|
2904
5307
|
}
|
|
5308
|
+
async applyLearnedReranker(input) {
|
|
5309
|
+
if (!this.rerankerProvider || !this.rerankerDescriptor) {
|
|
5310
|
+
return input.candidates;
|
|
5311
|
+
}
|
|
5312
|
+
const cappedTopN = Math.min(this.rerankerTopN, input.candidates.length);
|
|
5313
|
+
if (cappedTopN <= 1) {
|
|
5314
|
+
return input.candidates;
|
|
5315
|
+
}
|
|
5316
|
+
const head = input.candidates.slice(0, cappedTopN);
|
|
5317
|
+
const tail = input.candidates.slice(cappedTopN);
|
|
5318
|
+
const labels = {
|
|
5319
|
+
provider: this.rerankerDescriptor.provider,
|
|
5320
|
+
model: this.rerankerDescriptor.model ?? "unknown"
|
|
5321
|
+
};
|
|
5322
|
+
this.observability.metrics.increment("retrieval_reranker_requests_total", 1, labels);
|
|
5323
|
+
const startedAt = Date.now();
|
|
5324
|
+
try {
|
|
5325
|
+
const reranked = await this.rerankerProvider.rerank({
|
|
5326
|
+
query: input.query,
|
|
5327
|
+
documents: head.map((candidate) => buildRerankerDocument(candidate)),
|
|
5328
|
+
top_n: cappedTopN
|
|
5329
|
+
});
|
|
5330
|
+
if (!Array.isArray(reranked) || reranked.length === 0) {
|
|
5331
|
+
throw new RerankerProviderRequestError("invalid_response", "reranker response must contain at least one result");
|
|
5332
|
+
}
|
|
5333
|
+
const seen = new Set();
|
|
5334
|
+
const reordered = [];
|
|
5335
|
+
for (const row of reranked) {
|
|
5336
|
+
if (!Number.isInteger(row.index)) {
|
|
5337
|
+
throw new RerankerProviderRequestError("invalid_response", "reranker result index must be an integer");
|
|
5338
|
+
}
|
|
5339
|
+
if (row.index < 0 || row.index >= head.length) {
|
|
5340
|
+
throw new RerankerProviderRequestError("invalid_response", "reranker result index out of range");
|
|
5341
|
+
}
|
|
5342
|
+
if (seen.has(row.index)) {
|
|
5343
|
+
continue;
|
|
5344
|
+
}
|
|
5345
|
+
const candidate = head[row.index];
|
|
5346
|
+
if (!candidate) {
|
|
5347
|
+
continue;
|
|
5348
|
+
}
|
|
5349
|
+
seen.add(row.index);
|
|
5350
|
+
reordered.push(candidate);
|
|
5351
|
+
}
|
|
5352
|
+
for (let index = 0; index < head.length; index += 1) {
|
|
5353
|
+
if (seen.has(index)) {
|
|
5354
|
+
continue;
|
|
5355
|
+
}
|
|
5356
|
+
const candidate = head[index];
|
|
5357
|
+
if (candidate) {
|
|
5358
|
+
reordered.push(candidate);
|
|
5359
|
+
}
|
|
5360
|
+
}
|
|
5361
|
+
if (reordered.length === 0) {
|
|
5362
|
+
throw new RerankerProviderRequestError("invalid_response", "reranker did not return usable indexes");
|
|
5363
|
+
}
|
|
5364
|
+
const maxTailScore = tail[0]?.score ?? Number.NEGATIVE_INFINITY;
|
|
5365
|
+
const maxHeadScore = head[0]?.score ?? 0;
|
|
5366
|
+
const scoreAnchor = Math.max(maxHeadScore, maxTailScore) + 1;
|
|
5367
|
+
const scoreStep = 1e-6;
|
|
5368
|
+
const adjusted = reordered.map((candidate, index) => ({
|
|
5369
|
+
...candidate,
|
|
5370
|
+
score: scoreAnchor - index * scoreStep
|
|
5371
|
+
}));
|
|
5372
|
+
return [...adjusted, ...tail];
|
|
5373
|
+
}
|
|
5374
|
+
catch (error) {
|
|
5375
|
+
const reason = classifyRerankerFailureReason(error);
|
|
5376
|
+
this.observability.metrics.increment("retrieval_reranker_failures_total", 1, {
|
|
5377
|
+
...labels,
|
|
5378
|
+
reason
|
|
5379
|
+
});
|
|
5380
|
+
this.observability.metrics.increment("retrieval_reranker_fallback_total", 1, {
|
|
5381
|
+
reason
|
|
5382
|
+
});
|
|
5383
|
+
this.observability.logger.warn("search_context reranker fallback applied", {
|
|
5384
|
+
trace_id: input.trace_id,
|
|
5385
|
+
provider: labels.provider,
|
|
5386
|
+
model: labels.model,
|
|
5387
|
+
reason,
|
|
5388
|
+
top_n: cappedTopN,
|
|
5389
|
+
error_message: error instanceof Error ? error.message : String(error)
|
|
5390
|
+
});
|
|
5391
|
+
return input.candidates;
|
|
5392
|
+
}
|
|
5393
|
+
finally {
|
|
5394
|
+
this.observability.metrics.observe("retrieval_reranker_latency_ms", Date.now() - startedAt, labels);
|
|
5395
|
+
}
|
|
5396
|
+
}
|
|
2905
5397
|
async searchContext(input) {
|
|
2906
5398
|
const searchStartedAt = Date.now();
|
|
2907
5399
|
const index = await this.store.getLatestReadyIndex({
|
|
@@ -2915,9 +5407,9 @@ export class RetrievalCore {
|
|
|
2915
5407
|
tenant_id: input.tenant_id,
|
|
2916
5408
|
index_id: index.index_id
|
|
2917
5409
|
});
|
|
2918
|
-
const topK = Math.min(input.request.top_k ?? 8, MAX_TOP_K);
|
|
2919
|
-
const candidatePoolTopK = Math.min(MAX_TOP_K, Math.max(topK * 4, 12));
|
|
2920
5410
|
const query = normalizeQuery(input.request.query);
|
|
5411
|
+
const topK = Math.min(input.request.top_k ?? 8, MAX_TOP_K);
|
|
5412
|
+
const candidatePoolTopK = Math.max(Math.max(topK * 4, 12), this.internalCandidateDepth);
|
|
2921
5413
|
if (!indexMetadata) {
|
|
2922
5414
|
this.observability.metrics.increment("retrieval_embedding_metadata_mismatch_total", 1, {
|
|
2923
5415
|
reason: "metadata_missing"
|
|
@@ -2977,12 +5469,20 @@ export class RetrievalCore {
|
|
|
2977
5469
|
throw new RetrievalError("UPSTREAM_FAILURE", `embedding provider returned query embedding dimensions ${queryEmbedding.length}; expected ${this.embeddingDescriptor.dimensions}`);
|
|
2978
5470
|
}
|
|
2979
5471
|
const queryTokens = tokenize(query);
|
|
5472
|
+
const searchLiterals = extractSearchLiterals(query);
|
|
5473
|
+
this.observability.metrics.observe("retrieval_candidate_depth_requested", topK, {
|
|
5474
|
+
retrieval_profile_id: this.scoringProfileId
|
|
5475
|
+
});
|
|
5476
|
+
this.observability.metrics.observe("retrieval_candidate_depth_effective", candidatePoolTopK, {
|
|
5477
|
+
retrieval_profile_id: this.scoringProfileId
|
|
5478
|
+
});
|
|
2980
5479
|
const cacheKey = buildQueryCacheKey({
|
|
2981
5480
|
workspace_id: input.workspace_id,
|
|
2982
5481
|
index_version: index.index_version,
|
|
2983
5482
|
query,
|
|
2984
5483
|
top_k: topK,
|
|
2985
|
-
filters: input.request.filters
|
|
5484
|
+
filters: input.request.filters,
|
|
5485
|
+
retrieval_variant: `${this.rerankerCacheVariant}|context_pack:${this.contextPackingConfig.enabled ? "on" : "off"}|context_pack_spans:${this.contextPackingConfig.max_spans_per_result}|context_pack_gap:${this.contextPackingConfig.max_gap_lines}|snippet_integrity:${this.snippetIntegrityConfig.enabled ? "on" : "off"}|snippet_integrity_gap:${this.snippetIntegrityConfig.max_contiguous_gap_lines}|snippet_integrity_langs:${this.snippetIntegrityConfig.target_languages.join(",")}|snippet_repair:${this.snippetIntegrityConfig.repair_enabled ? "on" : "off"}|snippet_repair_env:${this.snippetIntegrityConfig.repair_max_envelope_lines}|snippet_repair_chars:${this.snippetIntegrityConfig.repair_max_snippet_chars}|chunk_recursive:${this.chunkingConfig.recursive_semantic_chunking_enabled ? "on" : "off"}|chunk_semantic_gap:${this.chunkingConfig.semantic_merge_gap_lines}|chunk_semantic_span:${this.chunkingConfig.semantic_merge_max_span_lines}|chunk_comment_absorb:${this.chunkingConfig.comment_forward_absorb_enabled ? "on" : "off"}|chunk_embed_prefix:${this.chunkingConfig.embedding_context_prefix_enabled ? "on" : "off"}|smart_cutoff:${this.scoringConfig.rerank.smart_cutoff_enabled ? "on" : "off"}|smart_cutoff_min_k:${this.scoringConfig.rerank.smart_cutoff_min_k}|smart_cutoff_max_k:${this.scoringConfig.rerank.smart_cutoff_max_k}|smart_cutoff_min_score:${this.scoringConfig.rerank.smart_cutoff_min_score}|smart_cutoff_top_ratio:${this.scoringConfig.rerank.smart_cutoff_top_ratio}|smart_cutoff_delta_abs:${this.scoringConfig.rerank.smart_cutoff_delta_abs}`
|
|
2986
5486
|
});
|
|
2987
5487
|
const cached = await this.cache.get(cacheKey);
|
|
2988
5488
|
if (cached) {
|
|
@@ -2999,6 +5499,8 @@ export class RetrievalCore {
|
|
|
2999
5499
|
tenant_id: input.tenant_id,
|
|
3000
5500
|
workspace_id: input.workspace_id
|
|
3001
5501
|
}, async () => {
|
|
5502
|
+
let literalPathMatchCount = 0;
|
|
5503
|
+
let literalSnippetMatchCount = 0;
|
|
3002
5504
|
let ranked;
|
|
3003
5505
|
if (this.store.rankChunksByIndex) {
|
|
3004
5506
|
ranked = await this.store.rankChunksByIndex({
|
|
@@ -3017,11 +5519,21 @@ export class RetrievalCore {
|
|
|
3017
5519
|
.map((candidate) => {
|
|
3018
5520
|
let score = candidate.score;
|
|
3019
5521
|
score += pathQualityBias(candidate.path, queryTokens, this.scoringConfig, query);
|
|
5522
|
+
const literalBoost = applyLiteralBoost({
|
|
5523
|
+
path: candidate.path,
|
|
5524
|
+
snippet: candidate.snippet,
|
|
5525
|
+
literals: searchLiterals,
|
|
5526
|
+
path_bias: this.scoringConfig.path_bias
|
|
5527
|
+
});
|
|
5528
|
+
score += literalBoost.boost;
|
|
5529
|
+
literalPathMatchCount += literalBoost.path_matches;
|
|
5530
|
+
literalSnippetMatchCount += literalBoost.snippet_matches;
|
|
3020
5531
|
if (looksLowInformation(candidate.snippet)) {
|
|
3021
5532
|
score -= this.scoringConfig.rerank.low_information_penalty;
|
|
3022
5533
|
}
|
|
3023
5534
|
const reason = chooseReason({
|
|
3024
5535
|
lexical: candidate.lexical_score,
|
|
5536
|
+
literal_match: literalBoost.matched,
|
|
3025
5537
|
path_match: candidate.path_match,
|
|
3026
5538
|
recency_boosted: candidate.recency_boosted
|
|
3027
5539
|
});
|
|
@@ -3059,10 +5571,24 @@ export class RetrievalCore {
|
|
|
3059
5571
|
score -= candidateWeights.generated_penalty;
|
|
3060
5572
|
}
|
|
3061
5573
|
score += pathQualityBias(chunk.path, queryTokens, this.scoringConfig, query);
|
|
5574
|
+
const literalBoost = applyLiteralBoost({
|
|
5575
|
+
path: chunk.path,
|
|
5576
|
+
snippet: chunk.snippet,
|
|
5577
|
+
literals: searchLiterals,
|
|
5578
|
+
path_bias: this.scoringConfig.path_bias
|
|
5579
|
+
});
|
|
5580
|
+
score += literalBoost.boost;
|
|
5581
|
+
literalPathMatchCount += literalBoost.path_matches;
|
|
5582
|
+
literalSnippetMatchCount += literalBoost.snippet_matches;
|
|
3062
5583
|
if (looksLowInformation(chunk.snippet)) {
|
|
3063
5584
|
score -= this.scoringConfig.rerank.low_information_penalty;
|
|
3064
5585
|
}
|
|
3065
|
-
const reason = chooseReason({
|
|
5586
|
+
const reason = chooseReason({
|
|
5587
|
+
lexical: l,
|
|
5588
|
+
literal_match: literalBoost.matched,
|
|
5589
|
+
path_match: pathMatch,
|
|
5590
|
+
recency_boosted: recencyBoost
|
|
5591
|
+
});
|
|
3066
5592
|
return {
|
|
3067
5593
|
path: chunk.path,
|
|
3068
5594
|
start_line: chunk.start_line,
|
|
@@ -3078,18 +5604,59 @@ export class RetrievalCore {
|
|
|
3078
5604
|
channel: "hybrid",
|
|
3079
5605
|
retrieval_profile_id: this.scoringProfileId
|
|
3080
5606
|
});
|
|
5607
|
+
this.observability.metrics.observe("retrieval_candidates_pre_rerank_count", output.length, {
|
|
5608
|
+
retrieval_profile_id: this.scoringProfileId
|
|
5609
|
+
});
|
|
5610
|
+
if (literalPathMatchCount > 0) {
|
|
5611
|
+
this.observability.metrics.increment("retrieval_literal_boost_applied_total", literalPathMatchCount, {
|
|
5612
|
+
retrieval_profile_id: this.scoringProfileId,
|
|
5613
|
+
channel: "path"
|
|
5614
|
+
});
|
|
5615
|
+
}
|
|
5616
|
+
if (literalSnippetMatchCount > 0) {
|
|
5617
|
+
this.observability.metrics.increment("retrieval_literal_boost_applied_total", literalSnippetMatchCount, {
|
|
5618
|
+
retrieval_profile_id: this.scoringProfileId,
|
|
5619
|
+
channel: "snippet"
|
|
5620
|
+
});
|
|
5621
|
+
}
|
|
3081
5622
|
return output;
|
|
3082
5623
|
});
|
|
5624
|
+
const rerankedCandidates = await this.observability.tracing.withSpan("retrieval.learned_rerank", { trace_id: input.trace_id }, async () => this.applyLearnedReranker({
|
|
5625
|
+
trace_id: input.trace_id,
|
|
5626
|
+
query,
|
|
5627
|
+
candidates
|
|
5628
|
+
}));
|
|
5629
|
+
const consolidatedCandidates = await this.observability.tracing.withSpan("retrieval.overlap_merge", { trace_id: input.trace_id }, async () => mergeOverlappingCandidates(rerankedCandidates, this.scoringConfig.rerank));
|
|
5630
|
+
this.observability.metrics.observe("retrieval_candidates_post_overlap_merge_count", consolidatedCandidates.length, {
|
|
5631
|
+
retrieval_profile_id: this.scoringProfileId
|
|
5632
|
+
});
|
|
5633
|
+
const mergedCandidateCount = Math.max(0, rerankedCandidates.length - consolidatedCandidates.length);
|
|
5634
|
+
if (mergedCandidateCount > 0) {
|
|
5635
|
+
this.observability.metrics.increment("retrieval_overlap_candidates_merged_total", mergedCandidateCount, {
|
|
5636
|
+
retrieval_profile_id: this.scoringProfileId
|
|
5637
|
+
});
|
|
5638
|
+
}
|
|
5639
|
+
const cutoffCandidates = await this.observability.tracing.withSpan("retrieval.smart_cutoff", { trace_id: input.trace_id }, async () => applySmartCutoffCandidates(consolidatedCandidates, this.scoringConfig.rerank));
|
|
5640
|
+
if (this.scoringConfig.rerank.smart_cutoff_enabled) {
|
|
5641
|
+
this.observability.metrics.increment("retrieval_smart_cutoff_applied_total", 1, {
|
|
5642
|
+
retrieval_profile_id: this.scoringProfileId
|
|
5643
|
+
});
|
|
5644
|
+
const droppedCount = Math.max(0, consolidatedCandidates.length - cutoffCandidates.length);
|
|
5645
|
+
this.observability.metrics.increment("retrieval_smart_cutoff_drop_count", droppedCount, {
|
|
5646
|
+
retrieval_profile_id: this.scoringProfileId
|
|
5647
|
+
});
|
|
5648
|
+
}
|
|
3083
5649
|
const deduped = await this.observability.tracing.withSpan("retrieval.rerank", { trace_id: input.trace_id }, async () => {
|
|
3084
5650
|
const output = [];
|
|
3085
5651
|
const seen = new Set();
|
|
3086
5652
|
const pathCounts = new Map();
|
|
5653
|
+
const selectedRangesByPath = new Map();
|
|
3087
5654
|
const directoryCounts = new Map();
|
|
3088
5655
|
const extensionCounts = new Map();
|
|
3089
5656
|
const maxChunksPerPath = hasFileLookupIntent(queryTokens)
|
|
3090
5657
|
? this.scoringConfig.rerank.max_chunks_per_path_file_lookup
|
|
3091
5658
|
: this.scoringConfig.rerank.max_chunks_per_path_default;
|
|
3092
|
-
const available = [...
|
|
5659
|
+
const available = [...cutoffCandidates];
|
|
3093
5660
|
while (output.length < topK && available.length > 0) {
|
|
3094
5661
|
let bestIndex = -1;
|
|
3095
5662
|
let bestAdjustedScore = Number.NEGATIVE_INFINITY;
|
|
@@ -3107,6 +5674,12 @@ export class RetrievalCore {
|
|
|
3107
5674
|
if (pathCount >= maxChunksPerPath) {
|
|
3108
5675
|
continue;
|
|
3109
5676
|
}
|
|
5677
|
+
if (this.scoringConfig.rerank.merge_overlapping_chunks_enabled && pathCount > 0) {
|
|
5678
|
+
const selectedRanges = selectedRangesByPath.get(candidate.path) ?? [];
|
|
5679
|
+
if (isHeavilyOverlappingLineRange(candidate, selectedRanges)) {
|
|
5680
|
+
continue;
|
|
5681
|
+
}
|
|
5682
|
+
}
|
|
3110
5683
|
const directoryKey = parentDirectory(candidate.path).toLowerCase();
|
|
3111
5684
|
const extensionKey = fileExtension(candidate.path);
|
|
3112
5685
|
const adjustedScore = candidate.score -
|
|
@@ -3139,6 +5712,13 @@ export class RetrievalCore {
|
|
|
3139
5712
|
const selectedKey = `${selected.path}:${selected.start_line}:${selected.end_line}`;
|
|
3140
5713
|
seen.add(selectedKey);
|
|
3141
5714
|
pathCounts.set(selected.path, (pathCounts.get(selected.path) ?? 0) + 1);
|
|
5715
|
+
const selectedRanges = selectedRangesByPath.get(selected.path);
|
|
5716
|
+
if (selectedRanges) {
|
|
5717
|
+
selectedRanges.push({ start_line: selected.start_line, end_line: selected.end_line });
|
|
5718
|
+
}
|
|
5719
|
+
else {
|
|
5720
|
+
selectedRangesByPath.set(selected.path, [{ start_line: selected.start_line, end_line: selected.end_line }]);
|
|
5721
|
+
}
|
|
3142
5722
|
const selectedDirectory = parentDirectory(selected.path).toLowerCase();
|
|
3143
5723
|
const selectedExtension = fileExtension(selected.path);
|
|
3144
5724
|
directoryCounts.set(selectedDirectory, (directoryCounts.get(selectedDirectory) ?? 0) + 1);
|
|
@@ -3147,16 +5727,61 @@ export class RetrievalCore {
|
|
|
3147
5727
|
}
|
|
3148
5728
|
return output;
|
|
3149
5729
|
});
|
|
5730
|
+
const candidateRankByKey = new Map();
|
|
5731
|
+
for (let index = 0; index < cutoffCandidates.length; index += 1) {
|
|
5732
|
+
const candidate = cutoffCandidates[index];
|
|
5733
|
+
if (!candidate) {
|
|
5734
|
+
continue;
|
|
5735
|
+
}
|
|
5736
|
+
const key = `${candidate.path}:${candidate.start_line}:${candidate.end_line}`;
|
|
5737
|
+
if (!candidateRankByKey.has(key)) {
|
|
5738
|
+
candidateRankByKey.set(key, index + 1);
|
|
5739
|
+
}
|
|
5740
|
+
}
|
|
5741
|
+
let literalMatchesInTopK = 0;
|
|
5742
|
+
for (let postRank = 0; postRank < deduped.length; postRank += 1) {
|
|
5743
|
+
const row = deduped[postRank];
|
|
5744
|
+
if (!row) {
|
|
5745
|
+
continue;
|
|
5746
|
+
}
|
|
5747
|
+
if (isExactLiteralReason(row.reason)) {
|
|
5748
|
+
literalMatchesInTopK += 1;
|
|
5749
|
+
}
|
|
5750
|
+
this.observability.metrics.increment("retrieval_reason_topk_total", 1, {
|
|
5751
|
+
retrieval_profile_id: this.scoringProfileId,
|
|
5752
|
+
reason: row.reason
|
|
5753
|
+
});
|
|
5754
|
+
const key = `${row.path}:${row.start_line}:${row.end_line}`;
|
|
5755
|
+
const preRank = candidateRankByKey.get(key) ?? postRank + 1;
|
|
5756
|
+
this.observability.metrics.observe("retrieval_rank_shift_delta", preRank - (postRank + 1), {
|
|
5757
|
+
retrieval_profile_id: this.scoringProfileId
|
|
5758
|
+
});
|
|
5759
|
+
}
|
|
5760
|
+
this.observability.metrics.observe("retrieval_literal_matches_topk", literalMatchesInTopK, {
|
|
5761
|
+
retrieval_profile_id: this.scoringProfileId
|
|
5762
|
+
});
|
|
5763
|
+
const packedResults = packSearchResultsWithContext({
|
|
5764
|
+
selected: deduped,
|
|
5765
|
+
sourceCandidates: cutoffCandidates,
|
|
5766
|
+
config: this.contextPackingConfig
|
|
5767
|
+
});
|
|
5768
|
+
const assembledResults = annotateSearchResultsWithSnippetIntegrity({
|
|
5769
|
+
selected: packedResults,
|
|
5770
|
+
sourceCandidates: cutoffCandidates,
|
|
5771
|
+
config: this.snippetIntegrityConfig,
|
|
5772
|
+
observability: this.observability,
|
|
5773
|
+
retrievalProfileId: this.scoringProfileId
|
|
5774
|
+
});
|
|
3150
5775
|
const output = {
|
|
3151
5776
|
trace_id: input.trace_id,
|
|
3152
|
-
results:
|
|
5777
|
+
results: assembledResults,
|
|
3153
5778
|
search_metadata: {
|
|
3154
5779
|
latency_ms: Date.now() - searchStartedAt,
|
|
3155
5780
|
retrieval_mode: "hybrid",
|
|
3156
5781
|
index_version: index.index_version
|
|
3157
5782
|
}
|
|
3158
5783
|
};
|
|
3159
|
-
this.observability.metrics.observe("retrieval_topk_hit_proxy",
|
|
5784
|
+
this.observability.metrics.observe("retrieval_topk_hit_proxy", assembledResults.length > 0 ? 1 : 0, {
|
|
3160
5785
|
retrieval_profile_id: this.scoringProfileId
|
|
3161
5786
|
});
|
|
3162
5787
|
this.observability.logger.info("search_context completed", {
|
|
@@ -3171,18 +5796,117 @@ export class RetrievalCore {
|
|
|
3171
5796
|
await this.cache.set(cacheKey, output, this.cacheTtlSeconds);
|
|
3172
5797
|
return output;
|
|
3173
5798
|
}
|
|
5799
|
+
enhancerProviderLabels() {
|
|
5800
|
+
return {
|
|
5801
|
+
provider: this.enhancerProviderDescriptor?.provider ?? "template",
|
|
5802
|
+
model: this.enhancerProviderDescriptor?.model ?? "n/a",
|
|
5803
|
+
tool_mode: this.enhancerGenerationConfig.tool_mode
|
|
5804
|
+
};
|
|
5805
|
+
}
|
|
5806
|
+
buildEnhancerContextSnippets(results) {
|
|
5807
|
+
const maxSnippets = this.enhancerGenerationConfig.max_context_snippets;
|
|
5808
|
+
const snippetCharLimit = this.contextPackingConfig.enabled ? this.contextPackingConfig.enhancer_snippet_char_limit : 1_600;
|
|
5809
|
+
const snippets = [];
|
|
5810
|
+
for (const result of results.slice(0, maxSnippets)) {
|
|
5811
|
+
snippets.push({
|
|
5812
|
+
path: result.path,
|
|
5813
|
+
start_line: result.start_line,
|
|
5814
|
+
end_line: result.end_line,
|
|
5815
|
+
reason: result.reason,
|
|
5816
|
+
snippet: result.snippet.slice(0, snippetCharLimit),
|
|
5817
|
+
score: result.score
|
|
5818
|
+
});
|
|
5819
|
+
}
|
|
5820
|
+
return snippets;
|
|
5821
|
+
}
|
|
5822
|
+
async generateEnhancedPrompt(input) {
|
|
5823
|
+
if (!this.enhancerProvider) {
|
|
5824
|
+
return formatEnhancedPrompt({
|
|
5825
|
+
style: input.style_resolved,
|
|
5826
|
+
language: input.language,
|
|
5827
|
+
original_prompt: input.request.prompt,
|
|
5828
|
+
refs: input.context_refs
|
|
5829
|
+
});
|
|
5830
|
+
}
|
|
5831
|
+
const maxAttempts = this.enhancerGenerationConfig.max_retries + 1;
|
|
5832
|
+
let lastFailure;
|
|
5833
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
5834
|
+
const startedAt = Date.now();
|
|
5835
|
+
this.observability.metrics.increment("enhancer_provider_requests_total", 1, this.enhancerProviderLabels());
|
|
5836
|
+
try {
|
|
5837
|
+
const generated = await runWithInactivityTimeout({
|
|
5838
|
+
timeout_ms: this.enhancerGenerationConfig.timeout_ms,
|
|
5839
|
+
fn: ({ touch, signal }) => this.enhancerProvider.generate({
|
|
5840
|
+
trace_id: input.trace_id,
|
|
5841
|
+
tenant_id: input.tenant_id,
|
|
5842
|
+
workspace_id: input.workspace_id,
|
|
5843
|
+
request: input.request,
|
|
5844
|
+
style_requested: input.style_requested,
|
|
5845
|
+
style_resolved: input.style_resolved,
|
|
5846
|
+
intent: input.intent,
|
|
5847
|
+
query_intent: input.query_intent,
|
|
5848
|
+
language: input.language,
|
|
5849
|
+
context_refs: input.context_refs,
|
|
5850
|
+
context_snippets: input.context_snippets,
|
|
5851
|
+
warnings: input.warnings,
|
|
5852
|
+
questions: input.questions,
|
|
5853
|
+
tool_mode: this.enhancerGenerationConfig.tool_mode,
|
|
5854
|
+
abort_signal: signal,
|
|
5855
|
+
on_progress: touch
|
|
5856
|
+
})
|
|
5857
|
+
});
|
|
5858
|
+
this.observability.metrics.observe("enhancer_provider_latency_ms", Date.now() - startedAt, this.enhancerProviderLabels());
|
|
5859
|
+
const enhancedPrompt = normalizeProviderEnhancedPrompt(generated.enhanced_prompt);
|
|
5860
|
+
if (enhancedPrompt.length === 0) {
|
|
5861
|
+
throw new EnhancerProviderRequestError("invalid_response", "enhancer provider returned an empty enhanced_prompt");
|
|
5862
|
+
}
|
|
5863
|
+
return enhancedPrompt;
|
|
5864
|
+
}
|
|
5865
|
+
catch (error) {
|
|
5866
|
+
const reason = classifyEnhancerGenerationFailureReason(error);
|
|
5867
|
+
const failure = error instanceof EnhancerProviderRequestError ? error : new EnhancerProviderRequestError(reason, String(error));
|
|
5868
|
+
lastFailure = failure;
|
|
5869
|
+
this.observability.metrics.increment("enhancer_provider_failures_total", 1, {
|
|
5870
|
+
...this.enhancerProviderLabels(),
|
|
5871
|
+
reason
|
|
5872
|
+
});
|
|
5873
|
+
this.observability.logger.warn("enhancer provider generation failed", {
|
|
5874
|
+
trace_id: input.trace_id,
|
|
5875
|
+
attempt,
|
|
5876
|
+
max_attempts: maxAttempts,
|
|
5877
|
+
reason,
|
|
5878
|
+
retrying: attempt < maxAttempts &&
|
|
5879
|
+
reason !== "timeout" &&
|
|
5880
|
+
reason !== "schema_error" &&
|
|
5881
|
+
reason !== "invalid_response",
|
|
5882
|
+
style_requested: input.style_requested,
|
|
5883
|
+
style_resolved: input.style_resolved,
|
|
5884
|
+
provider: this.enhancerProviderDescriptor?.provider ?? "custom",
|
|
5885
|
+
model: this.enhancerProviderDescriptor?.model ?? "unknown",
|
|
5886
|
+
error: failure.message
|
|
5887
|
+
});
|
|
5888
|
+
if (reason === "timeout" || reason === "schema_error" || reason === "invalid_response") {
|
|
5889
|
+
break;
|
|
5890
|
+
}
|
|
5891
|
+
}
|
|
5892
|
+
}
|
|
5893
|
+
const message = lastFailure?.message ?? "enhancer provider failed";
|
|
5894
|
+
throw new RetrievalError("UPSTREAM_FAILURE", `enhancer provider failed after retries: ${message}`);
|
|
5895
|
+
}
|
|
3174
5896
|
async enhancePrompt(input) {
|
|
3175
5897
|
const startedAt = Date.now();
|
|
3176
5898
|
const warnings = [];
|
|
3177
|
-
const questions = [];
|
|
3178
|
-
const addQuestion = (value) => {
|
|
3179
|
-
if (!questions.includes(value)) {
|
|
3180
|
-
questions.push(value);
|
|
3181
|
-
}
|
|
3182
|
-
};
|
|
3183
5899
|
const intent = classifyIntent(input.request.prompt);
|
|
3184
5900
|
const queryIntent = classifyEnhancerQueryIntent(input.request.prompt, input.request.conversation_history);
|
|
3185
5901
|
const language = detectDominantLanguage(input.request.prompt, input.request.conversation_history);
|
|
5902
|
+
const style = resolveEnhancerPromptStyle({
|
|
5903
|
+
requested: input.request.style,
|
|
5904
|
+
intent,
|
|
5905
|
+
query_intent: queryIntent,
|
|
5906
|
+
prompt: input.request.prompt,
|
|
5907
|
+
history: input.request.conversation_history,
|
|
5908
|
+
has_context: Boolean(input.request.project_root_path && input.workspace_id)
|
|
5909
|
+
});
|
|
3186
5910
|
const negativePreferences = detectNegativePathPreferences(`${input.request.prompt}\n${input.request.conversation_history.map((entry) => entry.content).join("\n")}`);
|
|
3187
5911
|
const intentPolicy = resolveEnhancerIntentPolicy({
|
|
3188
5912
|
query_intent: queryIntent,
|
|
@@ -3230,7 +5954,7 @@ export class RetrievalCore {
|
|
|
3230
5954
|
top_k: MAX_TOP_K
|
|
3231
5955
|
}
|
|
3232
5956
|
});
|
|
3233
|
-
const budgetedResults = trimToContextBudget(retrieval.results);
|
|
5957
|
+
const budgetedResults = trimToContextBudget(retrieval.results, this.contextPackingConfig.enabled ? "lightweight" : "ranking");
|
|
3234
5958
|
const dedupedByPath = dedupeEnhancerCandidatesByPath(budgetedResults);
|
|
3235
5959
|
const collapsedByDirectory = collapseEnhancerCandidatesByDirectory(dedupedByPath, intentPolicy.max_candidates_per_directory_pre_rerank);
|
|
3236
5960
|
const filteredCandidates = applyEnhancerIntentPathFiltering(collapsedByDirectory, {
|
|
@@ -3292,16 +6016,6 @@ export class RetrievalCore {
|
|
|
3292
6016
|
strict_impl_only_filtering: intentPolicy.strict_impl_only_filtering
|
|
3293
6017
|
}));
|
|
3294
6018
|
searchResults = collapseEnhancerCandidatesByDirectory(searchResults, intentPolicy.max_candidates_per_directory_pre_rerank).slice(0, intentPolicy.max_candidates_pre_rerank);
|
|
3295
|
-
const symbolCandidates = extractLikelyCodeSymbols(`${input.request.prompt}\n${input.request.conversation_history.map((entry) => entry.content).join("\n")}`, 3);
|
|
3296
|
-
if (confidenceSignals.failed_signals.includes("score_spread")) {
|
|
3297
|
-
addQuestion(localizeLowConfidenceQuestion({ language, kind: "scope" }));
|
|
3298
|
-
}
|
|
3299
|
-
if (confidenceSignals.failed_signals.includes("token_overlap")) {
|
|
3300
|
-
addQuestion(localizeLowConfidenceQuestion({ language, kind: "symbol", symbol: symbolCandidates[0] }));
|
|
3301
|
-
}
|
|
3302
|
-
if (confidenceSignals.failed_signals.includes("path_diversity")) {
|
|
3303
|
-
addQuestion(localizeLowConfidenceQuestion({ language, kind: "source_priority" }));
|
|
3304
|
-
}
|
|
3305
6019
|
}
|
|
3306
6020
|
else {
|
|
3307
6021
|
searchResults = dedupeEnhancerCandidatesByPath(searchResults);
|
|
@@ -3310,6 +6024,9 @@ export class RetrievalCore {
|
|
|
3310
6024
|
candidateCountPostRerank = searchResults.length;
|
|
3311
6025
|
}
|
|
3312
6026
|
catch (error) {
|
|
6027
|
+
if (error instanceof RetrievalError && error.code === "RATE_LIMITED") {
|
|
6028
|
+
throw error;
|
|
6029
|
+
}
|
|
3313
6030
|
warnings.push("Context retrieval unavailable; enhancement generated with limited confidence.");
|
|
3314
6031
|
fallbackTriggered = true;
|
|
3315
6032
|
fallbackReason = "context_retrieval_unavailable";
|
|
@@ -3319,31 +6036,34 @@ export class RetrievalCore {
|
|
|
3319
6036
|
}
|
|
3320
6037
|
}
|
|
3321
6038
|
}
|
|
3322
|
-
if (intent === "unknown") {
|
|
3323
|
-
addQuestion(language === "es"
|
|
3324
|
-
? "¿Cuál es el resultado esperado exacto y el alcance del cambio?"
|
|
3325
|
-
: language === "zh"
|
|
3326
|
-
? "这次变更的精确目标和范围是什么?"
|
|
3327
|
-
: "What exact outcome and scope should this change target?");
|
|
3328
|
-
}
|
|
3329
6039
|
const contextRefs = searchResults.map((result) => ({
|
|
3330
6040
|
path: result.path,
|
|
3331
6041
|
start_line: result.start_line,
|
|
3332
6042
|
end_line: result.end_line,
|
|
3333
6043
|
reason: result.reason
|
|
3334
6044
|
}));
|
|
3335
|
-
const
|
|
6045
|
+
const contextSnippets = this.buildEnhancerContextSnippets(searchResults);
|
|
6046
|
+
const enhancedPrompt = await this.generateEnhancedPrompt({
|
|
6047
|
+
trace_id: input.trace_id,
|
|
6048
|
+
tenant_id: input.tenant_id,
|
|
6049
|
+
workspace_id: input.workspace_id,
|
|
6050
|
+
request: input.request,
|
|
6051
|
+
style_requested: style.requested,
|
|
6052
|
+
style_resolved: style.resolved,
|
|
3336
6053
|
intent,
|
|
6054
|
+
query_intent: queryIntent,
|
|
3337
6055
|
language,
|
|
3338
|
-
|
|
3339
|
-
|
|
6056
|
+
context_refs: contextRefs,
|
|
6057
|
+
context_snippets: contextSnippets,
|
|
6058
|
+
warnings: [],
|
|
6059
|
+
questions: []
|
|
3340
6060
|
});
|
|
3341
6061
|
const output = {
|
|
3342
6062
|
trace_id: input.trace_id,
|
|
3343
6063
|
enhanced_prompt: enhancedPrompt,
|
|
3344
6064
|
context_refs: contextRefs,
|
|
3345
|
-
warnings,
|
|
3346
|
-
questions
|
|
6065
|
+
warnings: [],
|
|
6066
|
+
questions: []
|
|
3347
6067
|
};
|
|
3348
6068
|
const latency_ms = Date.now() - startedAt;
|
|
3349
6069
|
this.observability.metrics.observe("enhancer_latency_ms", latency_ms, {});
|
|
@@ -3392,6 +6112,11 @@ export class RetrievalCore {
|
|
|
3392
6112
|
fallback_triggered: fallbackTriggered,
|
|
3393
6113
|
fallback_reason: fallbackReason,
|
|
3394
6114
|
query_intent: queryIntent,
|
|
6115
|
+
style_requested: style.requested,
|
|
6116
|
+
style_resolved: style.resolved,
|
|
6117
|
+
enhancer_provider: this.enhancerProviderDescriptor?.provider ?? "template",
|
|
6118
|
+
enhancer_model: this.enhancerProviderDescriptor?.model ?? null,
|
|
6119
|
+
enhancer_tool_mode: this.enhancerGenerationConfig.tool_mode,
|
|
3395
6120
|
confidence_score_spread: confidenceSignals?.score_spread ?? null,
|
|
3396
6121
|
confidence_token_overlap: confidenceSignals?.token_overlap ?? null,
|
|
3397
6122
|
confidence_path_diversity: confidenceSignals?.path_diversity ?? null,
|