@contextrail/code-review-agent 0.1.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +26 -0
- package/MODEL_RECOMMENDATIONS.md +178 -0
- package/README.md +177 -0
- package/dist/config/defaults.d.ts +72 -0
- package/dist/config/defaults.js +113 -0
- package/dist/config/index.d.ts +34 -0
- package/dist/config/index.js +89 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +603 -0
- package/dist/llm/factory.d.ts +21 -0
- package/dist/llm/factory.js +50 -0
- package/dist/llm/index.d.ts +3 -0
- package/dist/llm/index.js +2 -0
- package/dist/llm/service.d.ts +38 -0
- package/dist/llm/service.js +191 -0
- package/dist/llm/types.d.ts +119 -0
- package/dist/llm/types.js +1 -0
- package/dist/logging/logger.d.ts +9 -0
- package/dist/logging/logger.js +52 -0
- package/dist/mcp/client.d.ts +429 -0
- package/dist/mcp/client.js +173 -0
- package/dist/mcp/mcp-tools.d.ts +292 -0
- package/dist/mcp/mcp-tools.js +40 -0
- package/dist/mcp/token-validation.d.ts +31 -0
- package/dist/mcp/token-validation.js +57 -0
- package/dist/mcp/tools-provider.d.ts +18 -0
- package/dist/mcp/tools-provider.js +24 -0
- package/dist/observability/index.d.ts +2 -0
- package/dist/observability/index.js +1 -0
- package/dist/observability/metrics.d.ts +48 -0
- package/dist/observability/metrics.js +86 -0
- package/dist/orchestrator/agentic-orchestrator.d.ts +29 -0
- package/dist/orchestrator/agentic-orchestrator.js +136 -0
- package/dist/orchestrator/prompts.d.ts +25 -0
- package/dist/orchestrator/prompts.js +98 -0
- package/dist/orchestrator/validation.d.ts +2 -0
- package/dist/orchestrator/validation.js +7 -0
- package/dist/orchestrator/writer.d.ts +4 -0
- package/dist/orchestrator/writer.js +17 -0
- package/dist/output/aggregator.d.ts +30 -0
- package/dist/output/aggregator.js +132 -0
- package/dist/output/prompts.d.ts +32 -0
- package/dist/output/prompts.js +153 -0
- package/dist/output/schema.d.ts +1515 -0
- package/dist/output/schema.js +224 -0
- package/dist/output/writer.d.ts +31 -0
- package/dist/output/writer.js +120 -0
- package/dist/review-inputs/chunking.d.ts +29 -0
- package/dist/review-inputs/chunking.js +113 -0
- package/dist/review-inputs/diff-summary.d.ts +52 -0
- package/dist/review-inputs/diff-summary.js +83 -0
- package/dist/review-inputs/file-patterns.d.ts +40 -0
- package/dist/review-inputs/file-patterns.js +182 -0
- package/dist/review-inputs/filtering.d.ts +31 -0
- package/dist/review-inputs/filtering.js +53 -0
- package/dist/review-inputs/git-diff-provider.d.ts +2 -0
- package/dist/review-inputs/git-diff-provider.js +42 -0
- package/dist/review-inputs/index.d.ts +46 -0
- package/dist/review-inputs/index.js +122 -0
- package/dist/review-inputs/path-validation.d.ts +10 -0
- package/dist/review-inputs/path-validation.js +37 -0
- package/dist/review-inputs/surrounding-context.d.ts +35 -0
- package/dist/review-inputs/surrounding-context.js +180 -0
- package/dist/review-inputs/triage.d.ts +57 -0
- package/dist/review-inputs/triage.js +81 -0
- package/dist/reviewers/executor.d.ts +41 -0
- package/dist/reviewers/executor.js +357 -0
- package/dist/reviewers/findings-merge.d.ts +9 -0
- package/dist/reviewers/findings-merge.js +131 -0
- package/dist/reviewers/iteration.d.ts +17 -0
- package/dist/reviewers/iteration.js +95 -0
- package/dist/reviewers/persistence.d.ts +17 -0
- package/dist/reviewers/persistence.js +55 -0
- package/dist/reviewers/progress-tracker.d.ts +115 -0
- package/dist/reviewers/progress-tracker.js +194 -0
- package/dist/reviewers/prompt.d.ts +42 -0
- package/dist/reviewers/prompt.js +246 -0
- package/dist/reviewers/tool-call-tracker.d.ts +18 -0
- package/dist/reviewers/tool-call-tracker.js +40 -0
- package/dist/reviewers/types.d.ts +12 -0
- package/dist/reviewers/types.js +1 -0
- package/dist/reviewers/validation-rules.d.ts +27 -0
- package/dist/reviewers/validation-rules.js +189 -0
- package/package.json +79 -0
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { chunkReviewInputs } from '../review-inputs/chunking.js';
|
|
3
|
+
import { filterDiffInputsByPatterns, filterFilesByPatterns, } from '../review-inputs/file-patterns.js';
|
|
4
|
+
import { isDiffInputs } from '../review-inputs/index.js';
|
|
5
|
+
import { DEFAULT_MAX_ITERATIONS, DEFAULT_MAX_STEPS } from '../config/defaults.js';
|
|
6
|
+
import { initProgress, updateProgress, appendFailure, formatTimestamp } from './progress-tracker.js';
|
|
7
|
+
import { buildPromptMessages } from './prompt.js';
|
|
8
|
+
import { runReviewerIteration } from './iteration.js';
|
|
9
|
+
import { logCompletion, logContinuation, logIteration, writeFindings, writeTokenMetrics } from './persistence.js';
|
|
10
|
+
import { mergeIterationTracking, mergeTokenUsage } from './tool-call-tracker.js';
|
|
11
|
+
import { mergeChunkFindings } from './findings-merge.js';
|
|
12
|
+
import { createLlmService } from '../llm/factory.js';
|
|
13
|
+
const formatIssuePath = (pathSegments) => {
|
|
14
|
+
if (!Array.isArray(pathSegments) || pathSegments.length === 0) {
|
|
15
|
+
return '(root)';
|
|
16
|
+
}
|
|
17
|
+
return pathSegments.map((segment) => String(segment)).join('.');
|
|
18
|
+
};
|
|
19
|
+
const extractValidationIssues = (error) => {
|
|
20
|
+
const candidates = [];
|
|
21
|
+
const seen = new Set();
|
|
22
|
+
let current = error;
|
|
23
|
+
for (let i = 0; i < 4 && current && typeof current === 'object' && !seen.has(current); i++) {
|
|
24
|
+
seen.add(current);
|
|
25
|
+
candidates.push(current);
|
|
26
|
+
current = current.cause;
|
|
27
|
+
}
|
|
28
|
+
for (const candidate of candidates) {
|
|
29
|
+
const issues = candidate.issues;
|
|
30
|
+
if (!Array.isArray(issues)) {
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
return issues
|
|
34
|
+
.map((issue) => {
|
|
35
|
+
const entry = issue;
|
|
36
|
+
return {
|
|
37
|
+
path: formatIssuePath(entry.path ?? []),
|
|
38
|
+
message: entry.message ?? 'Unknown validation issue',
|
|
39
|
+
};
|
|
40
|
+
})
|
|
41
|
+
.slice(0, 10);
|
|
42
|
+
}
|
|
43
|
+
return [];
|
|
44
|
+
};
|
|
45
|
+
const buildFailureDetails = (error, fallbackDetails) => {
|
|
46
|
+
const details = [fallbackDetails];
|
|
47
|
+
if (error && typeof error === 'object') {
|
|
48
|
+
const name = error.name;
|
|
49
|
+
if (name) {
|
|
50
|
+
details.push(`Error type: ${name}`);
|
|
51
|
+
}
|
|
52
|
+
const causeMessage = error.cause?.message;
|
|
53
|
+
if (causeMessage) {
|
|
54
|
+
details.push(`Cause: ${causeMessage}`);
|
|
55
|
+
}
|
|
56
|
+
const issues = extractValidationIssues(error);
|
|
57
|
+
if (issues.length > 0) {
|
|
58
|
+
details.push('Schema validation issues:');
|
|
59
|
+
for (const issue of issues) {
|
|
60
|
+
details.push(`- ${issue.path}: ${issue.message}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const rawText = error.text;
|
|
64
|
+
if (rawText) {
|
|
65
|
+
const snippet = rawText.length > 4000 ? `${rawText.slice(0, 4000)}\n...[truncated]` : rawText;
|
|
66
|
+
details.push(`Model raw output:\n\`\`\`json\n${snippet}\n\`\`\``);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return details.join('\n\n');
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* Extract filePatterns from prompt metadata.
|
|
73
|
+
*
|
|
74
|
+
* @param metadata - Prompt metadata (may be undefined or have nested structure)
|
|
75
|
+
* @returns FilePatterns if found, undefined otherwise
|
|
76
|
+
*/
|
|
77
|
+
export function extractFilePatterns(metadata) {
|
|
78
|
+
if (!metadata || typeof metadata !== 'object') {
|
|
79
|
+
return undefined;
|
|
80
|
+
}
|
|
81
|
+
// Handle nested metadata.scope.filePatterns structure from YAML
|
|
82
|
+
const meta = metadata;
|
|
83
|
+
const scope = meta.scope;
|
|
84
|
+
if (scope && typeof scope === 'object') {
|
|
85
|
+
const filePatterns = scope.filePatterns;
|
|
86
|
+
if (filePatterns && typeof filePatterns === 'object') {
|
|
87
|
+
const patterns = filePatterns;
|
|
88
|
+
return {
|
|
89
|
+
include: Array.isArray(patterns.include) ? patterns.include : undefined,
|
|
90
|
+
exclude: Array.isArray(patterns.exclude) ? patterns.exclude : undefined,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
// Handle direct filePatterns structure
|
|
95
|
+
const directPatterns = meta.filePatterns;
|
|
96
|
+
if (directPatterns && typeof directPatterns === 'object') {
|
|
97
|
+
const patterns = directPatterns;
|
|
98
|
+
return {
|
|
99
|
+
include: Array.isArray(patterns.include) ? patterns.include : undefined,
|
|
100
|
+
exclude: Array.isArray(patterns.exclude) ? patterns.exclude : undefined,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
return undefined;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Run reviewer loop with self-validation.
|
|
107
|
+
* Each iteration is a fresh generateText call (no history bleed).
|
|
108
|
+
* Based on context://agentic/execution/loop-orchestration
|
|
109
|
+
*
|
|
110
|
+
* @param reviewer - Reviewer name (e.g., 'reviewer-security')
|
|
111
|
+
* @param inputs - Review inputs (files, diffs)
|
|
112
|
+
* @param understanding - Orchestrator's understanding of changes
|
|
113
|
+
* @param outputDir - Base output directory
|
|
114
|
+
* @param deps - Dependencies (MCP client, config)
|
|
115
|
+
* @returns Reviewer findings
|
|
116
|
+
*/
|
|
117
|
+
export const runReviewerLoop = async (reviewer, inputs, understanding, outputDir, deps) => {
|
|
118
|
+
const { mcpClient, config, logger } = deps;
|
|
119
|
+
const maxIterations = config.maxIterations ?? DEFAULT_MAX_ITERATIONS;
|
|
120
|
+
const reviewersDir = path.join(outputDir, 'reviewers');
|
|
121
|
+
const reviewerDir = path.join(reviewersDir, reviewer);
|
|
122
|
+
// Initialize progress for this reviewer
|
|
123
|
+
let progress = await initProgress(reviewersDir, reviewer);
|
|
124
|
+
// Track context IDs used across iterations
|
|
125
|
+
const contextIds = [];
|
|
126
|
+
const toolCalls = [];
|
|
127
|
+
// Track token usage per iteration (with chunk details)
|
|
128
|
+
const iterationTokenData = [];
|
|
129
|
+
const recordFailureAndBlock = async (message, details) => {
|
|
130
|
+
await appendFailure(reviewersDir, reviewer, {
|
|
131
|
+
gate: 'validation',
|
|
132
|
+
message,
|
|
133
|
+
details,
|
|
134
|
+
});
|
|
135
|
+
await writeTokenMetrics(reviewerDir, reviewer, iterationTokenData);
|
|
136
|
+
progress = await updateProgress(reviewersDir, reviewer, {
|
|
137
|
+
status: 'blocked',
|
|
138
|
+
completedAt: formatTimestamp(),
|
|
139
|
+
lastFailure: {
|
|
140
|
+
gate: 'validation',
|
|
141
|
+
message,
|
|
142
|
+
details,
|
|
143
|
+
},
|
|
144
|
+
evidence: [`${reviewerDir}/failures.md`, `${reviewerDir}/progress.json`],
|
|
145
|
+
});
|
|
146
|
+
};
|
|
147
|
+
// Fetch reviewer-specific prompt from MCP
|
|
148
|
+
const promptResult = await mcpClient.getPrompt({ name: reviewer });
|
|
149
|
+
const promptMessages = buildPromptMessages(promptResult);
|
|
150
|
+
// Extract filePatterns from prompt metadata
|
|
151
|
+
const filePatterns = extractFilePatterns(promptResult.metadata);
|
|
152
|
+
// Filter inputs based on filePatterns from prompt metadata
|
|
153
|
+
let filteredInputs = inputs;
|
|
154
|
+
if (filePatterns) {
|
|
155
|
+
if (isDiffInputs(inputs)) {
|
|
156
|
+
const filtered = filterDiffInputsByPatterns(inputs.files, inputs.diffs, filePatterns);
|
|
157
|
+
filteredInputs = {
|
|
158
|
+
...inputs,
|
|
159
|
+
files: filtered.files,
|
|
160
|
+
diffs: filtered.diffs,
|
|
161
|
+
};
|
|
162
|
+
// Also filter context if present
|
|
163
|
+
if (inputs.context) {
|
|
164
|
+
const filteredContext = {};
|
|
165
|
+
for (const file of filtered.files) {
|
|
166
|
+
if (file in inputs.context && inputs.context[file]) {
|
|
167
|
+
filteredContext[file] = inputs.context[file];
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
filteredInputs.context = filteredContext;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
// File list mode - filter files and context
|
|
175
|
+
const filteredFiles = filterFilesByPatterns(inputs.files, filePatterns);
|
|
176
|
+
filteredInputs = {
|
|
177
|
+
...inputs,
|
|
178
|
+
files: filteredFiles,
|
|
179
|
+
};
|
|
180
|
+
// Also filter context if present
|
|
181
|
+
if (inputs.context) {
|
|
182
|
+
const filteredContext = {};
|
|
183
|
+
for (const file of filteredFiles) {
|
|
184
|
+
if (file in inputs.context && inputs.context[file]) {
|
|
185
|
+
filteredContext[file] = inputs.context[file];
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
filteredInputs.context = filteredContext;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
logger?.debug(`[${reviewer}] File pattern scope applied: include=${JSON.stringify(filePatterns.include ?? [])}, exclude=${JSON.stringify(filePatterns.exclude ?? [])}, before=${inputs.files.length}, after=${filteredInputs.files.length}`);
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
logger?.debug(`[${reviewer}] No file pattern scope found in prompt metadata; reviewing all ${inputs.files.length} files`);
|
|
195
|
+
}
|
|
196
|
+
// Create LLM service with MCP tools
|
|
197
|
+
const { service: llmService } = createLlmService({
|
|
198
|
+
openRouterApiKey: config.openRouterApiKey,
|
|
199
|
+
mcpClient,
|
|
200
|
+
logger,
|
|
201
|
+
});
|
|
202
|
+
const maxSteps = config.maxSteps ?? DEFAULT_MAX_STEPS;
|
|
203
|
+
let findings = null;
|
|
204
|
+
let iteration = 0;
|
|
205
|
+
// Chunk inputs if needed
|
|
206
|
+
const chunks = chunkReviewInputs(filteredInputs, config.chunking);
|
|
207
|
+
const isChunked = chunks.length > 1;
|
|
208
|
+
logger?.debug(`[${reviewer}] Execution plan: chunked=${isChunked}, chunks=${chunks.length}, maxIterations=${maxIterations}, reviewerModel=${config.reviewerModel}, criticModel=${config.criticModel ?? 'none'}`);
|
|
209
|
+
// Loop: review → confirm → re-review if needed
|
|
210
|
+
while (iteration < maxIterations) {
|
|
211
|
+
iteration += 1;
|
|
212
|
+
const iterationStartTime = formatTimestamp();
|
|
213
|
+
logger?.debug(`[${reviewer}] Iteration ${iteration}/${maxIterations} starting (criticPass=${iteration > 1 && findings !== null}, priorFindings=${findings?.findings.length ?? 0})`);
|
|
214
|
+
// Update progress to in-progress
|
|
215
|
+
progress = await updateProgress(reviewersDir, reviewer, {
|
|
216
|
+
status: 'in-progress',
|
|
217
|
+
attempts: progress.attempts + 1,
|
|
218
|
+
startedAt: progress.startedAt ?? iterationStartTime,
|
|
219
|
+
lastAttempt: iterationStartTime,
|
|
220
|
+
});
|
|
221
|
+
let iterationResult;
|
|
222
|
+
if (isChunked) {
|
|
223
|
+
// Process chunks sequentially and merge findings
|
|
224
|
+
const chunkFindings = [];
|
|
225
|
+
const chunkToolCalls = [];
|
|
226
|
+
const chunkContextIds = [];
|
|
227
|
+
const chunkUsages = [];
|
|
228
|
+
for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
|
|
229
|
+
const chunk = chunks[chunkIdx];
|
|
230
|
+
if (!chunk) {
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
logger?.debug(`[${reviewer}] Iteration ${iteration}: processing chunk ${chunkIdx + 1}/${chunks.length} (files=${chunk.files.length})`);
|
|
234
|
+
try {
|
|
235
|
+
const chunkResult = await runReviewerIteration(chunk, understanding, iteration, chunkIdx === 0 ? findings : null, // Only pass previous findings to first chunk
|
|
236
|
+
{
|
|
237
|
+
promptMessages,
|
|
238
|
+
llmService,
|
|
239
|
+
modelName: config.reviewerModel,
|
|
240
|
+
criticModelName: config.criticModel,
|
|
241
|
+
maxSteps,
|
|
242
|
+
reviewer,
|
|
243
|
+
logger,
|
|
244
|
+
}, config.prDescription, config.reviewDomains);
|
|
245
|
+
chunkFindings.push(chunkResult.findings);
|
|
246
|
+
mergeIterationTracking({ toolCalls: chunkToolCalls, contextIds: chunkContextIds }, chunkResult);
|
|
247
|
+
if (chunkResult.usage) {
|
|
248
|
+
chunkUsages.push({ chunkIndex: chunkIdx, usage: chunkResult.usage });
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
const errorMsg = error instanceof Error
|
|
253
|
+
? error.message
|
|
254
|
+
: `[REVIEWER] Failed to run reviewer iteration for chunk ${chunkIdx + 1}`;
|
|
255
|
+
const details = buildFailureDetails(error, `Chunk ${chunkIdx + 1} of ${chunks.length} failed`);
|
|
256
|
+
await recordFailureAndBlock(errorMsg, details);
|
|
257
|
+
throw error;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
// Merge findings from all chunks
|
|
261
|
+
const mergedFindings = mergeChunkFindings(chunkFindings);
|
|
262
|
+
findings = mergedFindings;
|
|
263
|
+
logger?.debug(`[${reviewer}] Iteration ${iteration}: merged chunk findings=${mergedFindings.findings.length}, validated=${mergedFindings.validated}`);
|
|
264
|
+
// Merge token usage from all chunks
|
|
265
|
+
const iterationUsage = mergeTokenUsage(chunkUsages.map((c) => c.usage));
|
|
266
|
+
iterationResult = {
|
|
267
|
+
findings: mergedFindings,
|
|
268
|
+
toolCalls: chunkToolCalls,
|
|
269
|
+
contextIds: chunkContextIds,
|
|
270
|
+
usage: iterationUsage,
|
|
271
|
+
};
|
|
272
|
+
// Track token usage for this iteration with chunk details
|
|
273
|
+
if (iterationUsage) {
|
|
274
|
+
iterationTokenData.push({
|
|
275
|
+
iteration,
|
|
276
|
+
usage: iterationUsage,
|
|
277
|
+
chunks: chunkUsages,
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
// Single chunk (or no chunking needed) - process normally
|
|
283
|
+
try {
|
|
284
|
+
iterationResult = await runReviewerIteration(inputs, understanding, iteration, findings, {
|
|
285
|
+
promptMessages,
|
|
286
|
+
llmService,
|
|
287
|
+
modelName: config.reviewerModel,
|
|
288
|
+
criticModelName: config.criticModel,
|
|
289
|
+
maxSteps,
|
|
290
|
+
reviewer,
|
|
291
|
+
logger,
|
|
292
|
+
}, config.prDescription, config.reviewDomains);
|
|
293
|
+
}
|
|
294
|
+
catch (error) {
|
|
295
|
+
const errorMsg = error instanceof Error ? error.message : '[REVIEWER] Failed to run reviewer iteration';
|
|
296
|
+
const details = buildFailureDetails(error, 'generateText returned no object output');
|
|
297
|
+
await recordFailureAndBlock(errorMsg, details);
|
|
298
|
+
throw error;
|
|
299
|
+
}
|
|
300
|
+
findings = iterationResult.findings;
|
|
301
|
+
}
|
|
302
|
+
mergeIterationTracking({ toolCalls, contextIds }, iterationResult);
|
|
303
|
+
// Track token usage for this iteration (non-chunked case)
|
|
304
|
+
if (!isChunked && iterationResult.usage) {
|
|
305
|
+
iterationTokenData.push({
|
|
306
|
+
iteration,
|
|
307
|
+
usage: iterationResult.usage,
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
// Log activity for this iteration
|
|
311
|
+
if (findings) {
|
|
312
|
+
await logIteration(reviewersDir, reviewer, iteration, findings, toolCalls, contextIds);
|
|
313
|
+
}
|
|
314
|
+
// Confirmation step: validate findings quality
|
|
315
|
+
if (findings && findings.validated) {
|
|
316
|
+
logger?.debug(`[${reviewer}] Validation complete at iteration ${iteration}: findings=${findings.findings.length}, toolCallsTracked=${toolCalls.length}, contextIdsTracked=${contextIds.length}`);
|
|
317
|
+
// Mark complete
|
|
318
|
+
progress = await updateProgress(reviewersDir, reviewer, {
|
|
319
|
+
status: 'done',
|
|
320
|
+
completedAt: formatTimestamp(),
|
|
321
|
+
evidence: [`${reviewerDir}/findings.json`, `${reviewerDir}/activity.md`],
|
|
322
|
+
});
|
|
323
|
+
await writeFindings(reviewerDir, findings);
|
|
324
|
+
await logCompletion(reviewersDir, reviewer, iteration, findings);
|
|
325
|
+
await writeTokenMetrics(reviewerDir, reviewer, iterationTokenData);
|
|
326
|
+
return findings;
|
|
327
|
+
}
|
|
328
|
+
// If not validated and we have more iterations, continue
|
|
329
|
+
if (iteration < maxIterations) {
|
|
330
|
+
logger?.debug(`[${reviewer}] Findings not validated at iteration ${iteration}; continuing to iteration ${iteration + 1}`);
|
|
331
|
+
await logContinuation(reviewersDir, reviewer, iteration + 1);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
// Max iterations reached without validation
|
|
335
|
+
const errorMsg = `Reviewer ${reviewer} reached max iterations (${maxIterations}) without validation`;
|
|
336
|
+
await appendFailure(reviewersDir, reviewer, {
|
|
337
|
+
gate: 'validation',
|
|
338
|
+
message: errorMsg,
|
|
339
|
+
details: `Findings were not validated after ${maxIterations} iterations`,
|
|
340
|
+
});
|
|
341
|
+
// Write findings anyway (may be partially complete)
|
|
342
|
+
if (findings) {
|
|
343
|
+
await writeFindings(reviewerDir, findings);
|
|
344
|
+
}
|
|
345
|
+
// Write token metrics even if max iterations reached
|
|
346
|
+
await writeTokenMetrics(reviewerDir, reviewer, iterationTokenData);
|
|
347
|
+
// Update progress to blocked
|
|
348
|
+
await updateProgress(reviewersDir, reviewer, {
|
|
349
|
+
status: 'blocked',
|
|
350
|
+
lastFailure: {
|
|
351
|
+
gate: 'validation',
|
|
352
|
+
message: errorMsg,
|
|
353
|
+
details: `Max iterations reached without validation`,
|
|
354
|
+
},
|
|
355
|
+
});
|
|
356
|
+
throw new Error(errorMsg);
|
|
357
|
+
};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ReviewerFindings } from '../output/schema.js';
|
|
2
|
+
/**
|
|
3
|
+
* Merge findings from multiple chunks deterministically.
|
|
4
|
+
* Findings are deduplicated based on file, line, and title.
|
|
5
|
+
*
|
|
6
|
+
* @param chunkFindings - Array of findings from each chunk
|
|
7
|
+
* @returns Merged findings
|
|
8
|
+
*/
|
|
9
|
+
export declare const mergeChunkFindings: (chunkFindings: ReviewerFindings[]) => ReviewerFindings;
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Get severity order value, defaulting to 'info' if invalid.
|
|
3
|
+
*/
|
|
4
|
+
const getSeverityOrder = (severity) => {
|
|
5
|
+
const severityOrder = {
|
|
6
|
+
critical: 4,
|
|
7
|
+
major: 3,
|
|
8
|
+
minor: 2,
|
|
9
|
+
info: 1,
|
|
10
|
+
};
|
|
11
|
+
return severityOrder[severity ?? 'info'] ?? 1;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Merge findings from multiple chunks deterministically.
|
|
15
|
+
* Findings are deduplicated based on file, line, and title.
|
|
16
|
+
*
|
|
17
|
+
* @param chunkFindings - Array of findings from each chunk
|
|
18
|
+
* @returns Merged findings
|
|
19
|
+
*/
|
|
20
|
+
export const mergeChunkFindings = (chunkFindings) => {
|
|
21
|
+
if (chunkFindings.length === 0) {
|
|
22
|
+
return {
|
|
23
|
+
findings: [],
|
|
24
|
+
validated: false,
|
|
25
|
+
notes: null,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (chunkFindings.length === 1) {
|
|
29
|
+
const singleChunk = chunkFindings[0];
|
|
30
|
+
if (!singleChunk) {
|
|
31
|
+
return {
|
|
32
|
+
findings: [],
|
|
33
|
+
validated: false,
|
|
34
|
+
notes: null,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
return singleChunk;
|
|
38
|
+
}
|
|
39
|
+
// Collect all findings
|
|
40
|
+
const allFindings = [];
|
|
41
|
+
for (const chunk of chunkFindings) {
|
|
42
|
+
allFindings.push(...chunk.findings);
|
|
43
|
+
}
|
|
44
|
+
// Deduplicate findings based on file, line, and title
|
|
45
|
+
// Use a map keyed by file+line+title for deduplication
|
|
46
|
+
const findingsMap = new Map();
|
|
47
|
+
for (const finding of allFindings) {
|
|
48
|
+
const key = `${finding.file ?? ''}:${finding.line ?? ''}:${finding.title}`;
|
|
49
|
+
const existing = findingsMap.get(key);
|
|
50
|
+
if (!existing) {
|
|
51
|
+
// New finding
|
|
52
|
+
findingsMap.set(key, finding);
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
// Merge context IDs if present
|
|
56
|
+
const mergedContextIds = new Set();
|
|
57
|
+
const mergedContextIdsViolated = new Set();
|
|
58
|
+
const mergedContextTitles = new Set();
|
|
59
|
+
if (existing.contextIdsUsed) {
|
|
60
|
+
existing.contextIdsUsed.forEach((id) => mergedContextIds.add(id));
|
|
61
|
+
}
|
|
62
|
+
if (finding.contextIdsUsed) {
|
|
63
|
+
finding.contextIdsUsed.forEach((id) => mergedContextIds.add(id));
|
|
64
|
+
}
|
|
65
|
+
if (existing.contextIdsViolated) {
|
|
66
|
+
existing.contextIdsViolated.forEach((id) => mergedContextIdsViolated.add(id));
|
|
67
|
+
}
|
|
68
|
+
if (finding.contextIdsViolated) {
|
|
69
|
+
finding.contextIdsViolated.forEach((id) => mergedContextIdsViolated.add(id));
|
|
70
|
+
}
|
|
71
|
+
if (existing.contextTitles) {
|
|
72
|
+
existing.contextTitles.forEach((title) => mergedContextTitles.add(title));
|
|
73
|
+
}
|
|
74
|
+
if (finding.contextTitles) {
|
|
75
|
+
finding.contextTitles.forEach((title) => mergedContextTitles.add(title));
|
|
76
|
+
}
|
|
77
|
+
// Keep the finding with higher severity (critical > major > minor > info)
|
|
78
|
+
const keepExisting = getSeverityOrder(existing.severity) >= getSeverityOrder(finding.severity);
|
|
79
|
+
if (!keepExisting) {
|
|
80
|
+
// Replace with new finding, but merge context IDs
|
|
81
|
+
// Azure workaround: use null instead of undefined for optional fields
|
|
82
|
+
findingsMap.set(key, {
|
|
83
|
+
...finding,
|
|
84
|
+
contextIdsUsed: mergedContextIds.size > 0 ? Array.from(mergedContextIds) : null,
|
|
85
|
+
contextIdsViolated: mergedContextIdsViolated.size > 0 ? Array.from(mergedContextIdsViolated) : null,
|
|
86
|
+
contextTitles: mergedContextTitles.size > 0 ? Array.from(mergedContextTitles) : null,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
// Update existing finding with merged context IDs
|
|
91
|
+
// Azure workaround: use null instead of undefined for optional fields
|
|
92
|
+
findingsMap.set(key, {
|
|
93
|
+
...existing,
|
|
94
|
+
contextIdsUsed: mergedContextIds.size > 0 ? Array.from(mergedContextIds) : null,
|
|
95
|
+
contextIdsViolated: mergedContextIdsViolated.size > 0 ? Array.from(mergedContextIdsViolated) : null,
|
|
96
|
+
contextTitles: mergedContextTitles.size > 0 ? Array.from(mergedContextTitles) : null,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// Convert map back to array
|
|
102
|
+
const mergedFindings = Array.from(findingsMap.values());
|
|
103
|
+
// Sort findings deterministically: by severity (desc), then by file, then by line
|
|
104
|
+
mergedFindings.sort((a, b) => {
|
|
105
|
+
const severityDiff = getSeverityOrder(b.severity) - getSeverityOrder(a.severity);
|
|
106
|
+
if (severityDiff !== 0) {
|
|
107
|
+
return severityDiff;
|
|
108
|
+
}
|
|
109
|
+
const fileA = a.file ?? '';
|
|
110
|
+
const fileB = b.file ?? '';
|
|
111
|
+
const fileDiff = fileA.localeCompare(fileB);
|
|
112
|
+
if (fileDiff !== 0) {
|
|
113
|
+
return fileDiff;
|
|
114
|
+
}
|
|
115
|
+
const lineA = a.line ?? 0;
|
|
116
|
+
const lineB = b.line ?? 0;
|
|
117
|
+
return lineA - lineB;
|
|
118
|
+
});
|
|
119
|
+
// Validated only if all chunks were validated
|
|
120
|
+
const allValidated = chunkFindings.every((chunk) => chunk.validated);
|
|
121
|
+
// Merge notes if present
|
|
122
|
+
const notes = chunkFindings
|
|
123
|
+
.map((chunk) => chunk.notes)
|
|
124
|
+
.filter((note) => Boolean(note))
|
|
125
|
+
.join('\n\n');
|
|
126
|
+
return {
|
|
127
|
+
findings: mergedFindings,
|
|
128
|
+
validated: allValidated,
|
|
129
|
+
notes: notes || null,
|
|
130
|
+
};
|
|
131
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { ReviewInputs } from '../review-inputs/index.js';
|
|
2
|
+
import type { ReviewerFindings } from '../output/schema.js';
|
|
3
|
+
import { type PromptMessage } from './prompt.js';
|
|
4
|
+
import type { ReviewerIterationResult } from './types.js';
|
|
5
|
+
import type { Logger } from '../logging/logger.js';
|
|
6
|
+
import type { LlmService } from '../llm/service.js';
|
|
7
|
+
type IterationDeps = {
|
|
8
|
+
promptMessages: PromptMessage[];
|
|
9
|
+
llmService: LlmService;
|
|
10
|
+
modelName: string;
|
|
11
|
+
criticModelName?: string;
|
|
12
|
+
maxSteps: number;
|
|
13
|
+
reviewer?: string;
|
|
14
|
+
logger?: Logger;
|
|
15
|
+
};
|
|
16
|
+
export declare const runReviewerIteration: (inputs: ReviewInputs, understanding: string, iteration: number, findings: ReviewerFindings | null, deps: IterationDeps, prDescription?: string, reviewDomains?: string[]) => Promise<ReviewerIterationResult>;
|
|
17
|
+
export {};
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { reviewerFindingsSchema, cleanFinding } from '../output/schema.js';
|
|
2
|
+
import { buildReviewerUserMessage } from './prompt.js';
|
|
3
|
+
import { collectToolCalls } from './tool-call-tracker.js';
|
|
4
|
+
import { applyAttributionGuardrails, applyDeterministicValidationRules } from './validation-rules.js';
|
|
5
|
+
export const runReviewerIteration = async (inputs, understanding, iteration, findings, deps, prDescription, reviewDomains) => {
|
|
6
|
+
const userMessage = buildReviewerUserMessage(inputs, understanding, iteration, findings, prDescription, reviewDomains);
|
|
7
|
+
const previousFindingsCount = findings?.findings.length ?? 0;
|
|
8
|
+
// Determine which model to use: critic model for iteration 2+, reviewer model for iteration 1
|
|
9
|
+
const isCriticPass = iteration > 1 && findings !== null;
|
|
10
|
+
const modelName = isCriticPass && deps.criticModelName ? deps.criticModelName : deps.modelName;
|
|
11
|
+
if (isCriticPass && deps.criticModelName) {
|
|
12
|
+
deps.logger?.info(`Critic pass starting (iteration ${iteration}, model: ${deps.criticModelName})`);
|
|
13
|
+
deps.logger?.debug(`Using critic model: ${deps.criticModelName} (iteration ${iteration})`);
|
|
14
|
+
deps.logger?.debug(`Critic evidence input (iteration ${iteration}): previous findings=${previousFindingsCount}, validated=${findings?.validated ?? false}`);
|
|
15
|
+
}
|
|
16
|
+
// Convert prompt messages to ModelMessage format
|
|
17
|
+
const messages = deps.promptMessages.map((msg) => ({
|
|
18
|
+
role: msg.role,
|
|
19
|
+
content: msg.content,
|
|
20
|
+
}));
|
|
21
|
+
// Add user message
|
|
22
|
+
messages.push({
|
|
23
|
+
role: 'user',
|
|
24
|
+
content: userMessage,
|
|
25
|
+
});
|
|
26
|
+
// Call LLM service with structured output
|
|
27
|
+
const result = await deps.llmService.generateStructuredOutput(messages, reviewerFindingsSchema, {
|
|
28
|
+
model: modelName,
|
|
29
|
+
maxSteps: deps.maxSteps,
|
|
30
|
+
metadata: {
|
|
31
|
+
operation: isCriticPass ? 'reviewer-critic' : 'reviewer-iteration',
|
|
32
|
+
model: modelName,
|
|
33
|
+
iteration,
|
|
34
|
+
reviewer: deps.reviewer,
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
let parsed = result.output;
|
|
38
|
+
// Normalize optional finding fields for stable downstream processing.
|
|
39
|
+
if (parsed.findings) {
|
|
40
|
+
parsed.findings = parsed.findings.map(cleanFinding);
|
|
41
|
+
}
|
|
42
|
+
// Apply non-negotiable attribution guardrails on every iteration.
|
|
43
|
+
const attributionResult = applyAttributionGuardrails(parsed.findings);
|
|
44
|
+
if (attributionResult.rejected.length > 0) {
|
|
45
|
+
deps.logger?.info(`Attribution guardrails rejected ${attributionResult.rejected.length} finding(s) in iteration ${iteration}`);
|
|
46
|
+
for (const { finding, reason } of attributionResult.rejected) {
|
|
47
|
+
deps.logger?.debug(`Attribution rejection: "${finding.title}" - ${reason}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
parsed = {
|
|
51
|
+
...parsed,
|
|
52
|
+
findings: attributionResult.passed,
|
|
53
|
+
notes: attributionResult.rejected.length > 0
|
|
54
|
+
? parsed.notes
|
|
55
|
+
? `${parsed.notes}\n\nAttribution guardrails removed ${attributionResult.rejected.length} finding(s).`
|
|
56
|
+
: `Attribution guardrails removed ${attributionResult.rejected.length} finding(s).`
|
|
57
|
+
: parsed.notes,
|
|
58
|
+
};
|
|
59
|
+
// Apply deterministic validation rules before LLM critic (for critic pass only)
|
|
60
|
+
if (isCriticPass) {
|
|
61
|
+
const beforeValidationCount = parsed.findings.length;
|
|
62
|
+
const validationResult = applyDeterministicValidationRules(parsed.findings);
|
|
63
|
+
if (validationResult.rejected.length > 0) {
|
|
64
|
+
deps.logger?.info(`Deterministic validation rejected ${validationResult.rejected.length} finding(s) in iteration ${iteration}`);
|
|
65
|
+
for (const { finding, reason } of validationResult.rejected) {
|
|
66
|
+
deps.logger?.debug(`Rejected finding: "${finding.title}" - ${reason}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
// Update findings with validated set
|
|
70
|
+
parsed = {
|
|
71
|
+
...parsed,
|
|
72
|
+
findings: validationResult.passed,
|
|
73
|
+
notes: parsed.notes
|
|
74
|
+
? `${parsed.notes}\n\nDeterministic validation removed ${validationResult.rejected.length} finding(s).`
|
|
75
|
+
: `Deterministic validation removed ${validationResult.rejected.length} finding(s).`,
|
|
76
|
+
};
|
|
77
|
+
deps.logger?.info(`Critic pass complete (iteration ${iteration}): ${validationResult.passed.length} finding(s) remaining`);
|
|
78
|
+
deps.logger?.debug(`Critic validation delta (iteration ${iteration}): before=${beforeValidationCount}, rejected=${validationResult.rejected.length}, after=${validationResult.passed.length}`);
|
|
79
|
+
}
|
|
80
|
+
const tracking = collectToolCalls(result.toolCalls);
|
|
81
|
+
const severityCounts = parsed.findings.reduce((acc, finding) => {
|
|
82
|
+
acc[finding.severity] += 1;
|
|
83
|
+
return acc;
|
|
84
|
+
}, { critical: 0, major: 0, minor: 0, info: 0, pass: 0 });
|
|
85
|
+
deps.logger?.debug(`Iteration ${iteration} evidence: findings=${parsed.findings.length}, validated=${parsed.validated}, toolCalls=${tracking.toolCalls.length}, contextIds=${tracking.contextIds.length}, severity=${JSON.stringify(severityCounts)}`);
|
|
86
|
+
if (result.usage) {
|
|
87
|
+
deps.logger?.debug(`Iteration ${iteration} token usage: prompt=${result.usage.promptTokens}, completion=${result.usage.completionTokens}, total=${result.usage.totalTokens}`);
|
|
88
|
+
}
|
|
89
|
+
return {
|
|
90
|
+
findings: parsed,
|
|
91
|
+
toolCalls: tracking.toolCalls,
|
|
92
|
+
contextIds: tracking.contextIds,
|
|
93
|
+
usage: result.usage,
|
|
94
|
+
};
|
|
95
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { ReviewerFindings, TokenUsage } from '../output/schema.js';
|
|
2
|
+
import type { ToolCallEntry } from './types.js';
|
|
3
|
+
export declare const logIteration: (reviewersDir: string, reviewer: string, iteration: number, findings: ReviewerFindings, toolCalls: ToolCallEntry[], contextIds: string[]) => Promise<void>;
|
|
4
|
+
export declare const logContinuation: (reviewersDir: string, reviewer: string, nextIteration: number) => Promise<void>;
|
|
5
|
+
export declare const writeFindings: (reviewerDir: string, findings: ReviewerFindings) => Promise<void>;
|
|
6
|
+
export declare const logCompletion: (reviewersDir: string, reviewer: string, iteration: number, findings: ReviewerFindings) => Promise<void>;
|
|
7
|
+
/**
|
|
8
|
+
* Write token usage metrics for a reviewer.
|
|
9
|
+
*/
|
|
10
|
+
export declare const writeTokenMetrics: (reviewerDir: string, reviewer: string, iterationTokenData: Array<{
|
|
11
|
+
iteration: number;
|
|
12
|
+
usage: TokenUsage;
|
|
13
|
+
chunks?: Array<{
|
|
14
|
+
chunkIndex: number;
|
|
15
|
+
usage: TokenUsage;
|
|
16
|
+
}>;
|
|
17
|
+
}>) => Promise<void>;
|