@effect-agent/pr-review 0.1.0-beta.21 → 0.1.0-beta.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -46
- package/dist/action.d.mts +13 -13
- package/dist/action.mjs +33 -25
- package/dist/action.mjs.map +1 -1
- package/dist/cli.mjs +2 -2
- package/dist/{fan-out-D5xrmadQ.d.mts → fan-out-BJBTAYuh.d.mts} +278 -303
- package/dist/{github-DSqZp3Ce.mjs → github-BbwYzNrC.mjs} +588 -243
- package/dist/github-BbwYzNrC.mjs.map +1 -0
- package/dist/index.d.mts +72 -116
- package/dist/index.mjs +11 -11
- package/dist/index.mjs.map +1 -1
- package/dist/{providers-CblG1G9b.mjs → providers-NyP-4rS6.mjs} +162 -578
- package/dist/providers-NyP-4rS6.mjs.map +1 -0
- package/dist/testing.d.mts +4 -15
- package/dist/testing.mjs +12 -41
- package/dist/testing.mjs.map +1 -1
- package/package.json +2 -2
- package/src/action.ts +40 -34
- package/src/internal/action-entry.ts +0 -1
- package/src/internal/coverage.ts +147 -532
- package/src/internal/factory.ts +29 -50
- package/src/internal/fan-out-scripted.ts +26 -80
- package/src/internal/fan-out.ts +577 -426
- package/src/internal/profiles.ts +8 -8
- package/src/internal/render.ts +34 -45
- package/src/internal/retirement.ts +3 -1
- package/src/internal/review-agent.ts +6 -1
- package/src/internal/review-state.ts +43 -19
- package/src/internal/review-units.ts +25 -0
- package/src/internal/run.ts +211 -174
- package/src/internal/source.ts +1 -1
- package/dist/github-DSqZp3Ce.mjs.map +0 -1
- package/dist/providers-CblG1G9b.mjs.map +0 -1
package/src/internal/coverage.ts
CHANGED
|
@@ -1,44 +1,24 @@
|
|
|
1
1
|
import { Option, Schema } from "effect";
|
|
2
2
|
import type { RunEvent } from "effect-agent";
|
|
3
3
|
|
|
4
|
-
import { anchorViolation } from "./anchors.ts";
|
|
5
4
|
import type { ChangedFile } from "./diff.ts";
|
|
6
5
|
import { isReviewableFile } from "./diff.ts";
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
FileReviewRequest,
|
|
10
|
-
FileReviewUnitResult,
|
|
11
|
-
ReviewCandidate,
|
|
12
|
-
reviewCandidateSubjectKey,
|
|
13
|
-
} from "./fan-out.ts";
|
|
14
|
-
import {
|
|
15
|
-
FileDiffView,
|
|
16
|
-
FileDiffQuery,
|
|
17
|
-
type ReviewConcern,
|
|
18
|
-
type ReviewFinding,
|
|
19
|
-
type WalkthroughEntry,
|
|
20
|
-
} from "./review-agent.ts";
|
|
21
|
-
import {
|
|
22
|
-
findingAnchorInUnitEvidence,
|
|
23
|
-
planReviewUnits,
|
|
24
|
-
type ReviewDiscoveryPass,
|
|
25
|
-
type ReviewUnit,
|
|
26
|
-
} from "./review-units.ts";
|
|
6
|
+
import { FileDiffView, FileDiffQuery } from "./review-agent.ts";
|
|
7
|
+
import type { ReviewUnitPlan } from "./review-units.ts";
|
|
27
8
|
|
|
28
9
|
// ---------------------------------------------------------------------------
|
|
29
10
|
// Two different claims are deliberately modeled:
|
|
30
11
|
//
|
|
31
12
|
// - input coverage: every required path was assigned bounded evidence or was
|
|
32
13
|
// explicitly reported outside the pipeline's capacity;
|
|
33
|
-
// - review assurance: every
|
|
34
|
-
// candidate-verification
|
|
14
|
+
// - review assurance: every scheduled discovery/specialist pass and every
|
|
15
|
+
// candidate-verification pass settled.
|
|
35
16
|
//
|
|
36
|
-
// Neither claims that the model found every defect.
|
|
17
|
+
// Neither claims that the model found every defect. The fan-out pipeline is
|
|
18
|
+
// host-scheduled (fan-out.ts), so its assurance is computed from direct pass
|
|
19
|
+
// results; only the flat reviewer is assessed from its Run event trace here.
|
|
37
20
|
// ---------------------------------------------------------------------------
|
|
38
21
|
|
|
39
|
-
export const ReviewShape = Schema.Literals(["flat", "fan-out"]);
|
|
40
|
-
export type ReviewShape = typeof ReviewShape.Type;
|
|
41
|
-
|
|
42
22
|
export class FailedReviewUnit extends Schema.Class<FailedReviewUnit>(
|
|
43
23
|
"@effect-agent/pr-review/FailedReviewUnit",
|
|
44
24
|
)({
|
|
@@ -87,6 +67,16 @@ export class ReviewInputCoverage extends Schema.Class<ReviewInputCoverage>(
|
|
|
87
67
|
unassignedPaths: Schema.Array(Schema.NonEmptyString.check(Schema.isMaxLength(512))).check(
|
|
88
68
|
Schema.isMaxLength(300),
|
|
89
69
|
),
|
|
70
|
+
/**
|
|
71
|
+
* Paths with neither a textual diff nor bounded base/head text (binaries,
|
|
72
|
+
* oversized files). Fail-closed: they keep the status incomplete for as
|
|
73
|
+
* long as they are part of the pull request — an unreviewable change must
|
|
74
|
+
* never authorize a green check. Exclude them deliberately with ignore
|
|
75
|
+
* globs when that is intended.
|
|
76
|
+
*/
|
|
77
|
+
undiffablePaths: Schema.Array(Schema.NonEmptyString.check(Schema.isMaxLength(512))).check(
|
|
78
|
+
Schema.isMaxLength(300),
|
|
79
|
+
),
|
|
90
80
|
reasons: Schema.Array(Schema.NonEmptyString.check(Schema.isMaxLength(1_000))).check(
|
|
91
81
|
Schema.isMaxLength(20),
|
|
92
82
|
),
|
|
@@ -100,6 +90,13 @@ export class FailedReviewPass extends Schema.Class<FailedReviewPass>(
|
|
|
100
90
|
errorTag: Schema.NonEmptyString.check(Schema.isMaxLength(256)),
|
|
101
91
|
}) {}
|
|
102
92
|
|
|
93
|
+
/**
|
|
94
|
+
* Settlement of scheduled review work. `incomplete` means reviewer-side work
|
|
95
|
+
* failed after its bounded retry — a machinery gap that is carried forward and
|
|
96
|
+
* retried on the next run, never a statement about the code under review.
|
|
97
|
+
* `unverified` is the flat reviewer's honest constant: one pass with no
|
|
98
|
+
* independent verifier is neither settled assurance nor a failure.
|
|
99
|
+
*/
|
|
103
100
|
export class ReviewAssurance extends Schema.Class<ReviewAssurance>(
|
|
104
101
|
"@effect-agent/pr-review/ReviewAssurance",
|
|
105
102
|
)({
|
|
@@ -114,7 +111,8 @@ export class ReviewAssurance extends Schema.Class<ReviewAssurance>(
|
|
|
114
111
|
confirmedCandidates: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
115
112
|
rejectedCandidates: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
116
113
|
unsettledCandidates: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
117
|
-
/**
|
|
114
|
+
/** Discovery claims discarded for anchors/paths outside their assigned evidence. */
|
|
115
|
+
discardedInvalidFindings: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
118
116
|
failedPasses: Schema.Array(FailedReviewPass).check(Schema.isMaxLength(64)),
|
|
119
117
|
reasons: Schema.Array(Schema.NonEmptyString.check(Schema.isMaxLength(1_000))).check(
|
|
120
118
|
Schema.isMaxLength(32),
|
|
@@ -142,7 +140,8 @@ const toolTrace = (events: ReadonlyArray<RunEvent>): ToolTrace => {
|
|
|
142
140
|
const sortedUnique = (values: Iterable<string>): ReadonlyArray<string> =>
|
|
143
141
|
[...new Set(values)].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0));
|
|
144
142
|
|
|
145
|
-
|
|
143
|
+
/** Render a bounded, deterministic "label (n): a, b, … (+k more)" reason line. */
|
|
144
|
+
export const boundedListReason = (label: string, values: Iterable<string>): string => {
|
|
146
145
|
const items = sortedUnique(values);
|
|
147
146
|
const prefix = `${label} (${items.length}): `;
|
|
148
147
|
let rendered = prefix;
|
|
@@ -160,12 +159,64 @@ const boundedListReason = (label: string, values: Iterable<string>): string => {
|
|
|
160
159
|
return rendered;
|
|
161
160
|
};
|
|
162
161
|
|
|
163
|
-
const
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
): ReviewInputCoverage =>
|
|
168
|
-
|
|
162
|
+
const anchorSurfaceAdjusted = (
|
|
163
|
+
inputCoverage: ReviewInputCoverage,
|
|
164
|
+
anchorFiles: ReadonlyArray<ChangedFile>,
|
|
165
|
+
totalAnchorFiles: number,
|
|
166
|
+
): ReviewInputCoverage =>
|
|
167
|
+
anchorFiles.length >= totalAnchorFiles
|
|
168
|
+
? inputCoverage
|
|
169
|
+
: ReviewInputCoverage.make({
|
|
170
|
+
...inputCoverage,
|
|
171
|
+
status: "incomplete",
|
|
172
|
+
reasons: [
|
|
173
|
+
...inputCoverage.reasons,
|
|
174
|
+
`full pull-request anchor surface exposed ${anchorFiles.length} of ${totalAnchorFiles} required files`,
|
|
175
|
+
],
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
/** The flat reviewer's honest constant assurance: one pass, no verifier. */
|
|
179
|
+
export const flatAssurance = (): ReviewAssurance =>
|
|
180
|
+
ReviewAssurance.make({
|
|
181
|
+
status: "unverified",
|
|
182
|
+
requiredGeneralDiscoveryPasses: 1,
|
|
183
|
+
completedGeneralDiscoveryPasses: 1,
|
|
184
|
+
requiredSpecialistPasses: 0,
|
|
185
|
+
completedSpecialistPasses: 0,
|
|
186
|
+
requiredVerificationPasses: 0,
|
|
187
|
+
completedVerificationPasses: 0,
|
|
188
|
+
discoveredCandidates: 0,
|
|
189
|
+
confirmedCandidates: 0,
|
|
190
|
+
rejectedCandidates: 0,
|
|
191
|
+
unsettledCandidates: 0,
|
|
192
|
+
discardedInvalidFindings: 0,
|
|
193
|
+
failedPasses: [],
|
|
194
|
+
reasons: [
|
|
195
|
+
"flat review has no independent candidate-verification pass; use the fan-out pipeline for a settled assurance result",
|
|
196
|
+
],
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
export interface FlatReviewAssessment {
|
|
200
|
+
readonly inputCoverage: ReviewInputCoverage;
|
|
201
|
+
readonly assurance: ReviewAssurance;
|
|
202
|
+
/** Retryable evidence gaps (failed or missing diff reads), never undiffable paths. */
|
|
203
|
+
readonly unreviewedPaths: ReadonlyArray<string>;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Assess one settled flat run from its Run event trace: which required paths
|
|
208
|
+
* received successful bounded diff evidence. This observes tool INPUT
|
|
209
|
+
* assignment only — the host cannot know which evidence the model weighed.
|
|
210
|
+
*/
|
|
211
|
+
export const assessFlatReview = (input: {
|
|
212
|
+
readonly files: ReadonlyArray<ChangedFile>;
|
|
213
|
+
readonly totalFiles: number;
|
|
214
|
+
readonly anchorFiles: ReadonlyArray<ChangedFile>;
|
|
215
|
+
readonly totalAnchorFiles: number;
|
|
216
|
+
readonly events: ReadonlyArray<RunEvent>;
|
|
217
|
+
}): FlatReviewAssessment => {
|
|
218
|
+
const trace = toolTrace(input.events);
|
|
219
|
+
const requiredPaths = sortedUnique(input.files.map((file) => file.path));
|
|
169
220
|
const assigned = new Set<string>();
|
|
170
221
|
const partial = new Set<string>();
|
|
171
222
|
const failedPaths = new Set<string>();
|
|
@@ -181,15 +232,19 @@ const flatInputCoverage = (
|
|
|
181
232
|
}
|
|
182
233
|
if (trace.failed.has(toolCallId)) failedPaths.add(query.value.path);
|
|
183
234
|
}
|
|
184
|
-
const undiffable =
|
|
235
|
+
const undiffable = new Set(
|
|
236
|
+
input.files.filter((file) => !isReviewableFile(file)).map((file) => file.path),
|
|
237
|
+
);
|
|
185
238
|
const unassigned = requiredPaths.filter(
|
|
186
|
-
(path) => !
|
|
239
|
+
(path) => !undiffable.has(path) && (!assigned.has(path) || failedPaths.has(path)),
|
|
187
240
|
);
|
|
188
241
|
const reasons: Array<string> = [];
|
|
189
|
-
if (files.length < totalFiles) {
|
|
190
|
-
reasons.push(
|
|
242
|
+
if (input.files.length < input.totalFiles) {
|
|
243
|
+
reasons.push(
|
|
244
|
+
`review range exposed ${input.files.length} of ${input.totalFiles} required files`,
|
|
245
|
+
);
|
|
191
246
|
}
|
|
192
|
-
if (undiffable.
|
|
247
|
+
if (undiffable.size > 0) {
|
|
193
248
|
reasons.push(
|
|
194
249
|
boundedListReason("required paths have no reviewable diff or bounded text", undiffable),
|
|
195
250
|
);
|
|
@@ -201,26 +256,50 @@ const flatInputCoverage = (
|
|
|
201
256
|
if (unassigned.length > 0) {
|
|
202
257
|
reasons.push(boundedListReason("required paths received no successful diff input", unassigned));
|
|
203
258
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
259
|
+
const inputCoverage = anchorSurfaceAdjusted(
|
|
260
|
+
ReviewInputCoverage.make({
|
|
261
|
+
status: reasons.length === 0 ? "complete" : "incomplete",
|
|
262
|
+
requiredPaths,
|
|
263
|
+
assignedPaths: sortedUnique(assigned),
|
|
264
|
+
partialPaths: sortedUnique(partial),
|
|
265
|
+
unassignedPaths: sortedUnique(unassigned),
|
|
266
|
+
undiffablePaths: sortedUnique(undiffable),
|
|
267
|
+
reasons,
|
|
268
|
+
}),
|
|
269
|
+
input.anchorFiles,
|
|
270
|
+
input.totalAnchorFiles,
|
|
271
|
+
);
|
|
272
|
+
return {
|
|
273
|
+
inputCoverage,
|
|
274
|
+
assurance: flatAssurance(),
|
|
275
|
+
// Everything still unreviewed and still part of the pull request carries
|
|
276
|
+
// forward — undiffable paths included, so the check stays fail-closed
|
|
277
|
+
// even after they leave the incremental delta.
|
|
278
|
+
unreviewedPaths: sortedUnique([...unassigned, ...undiffable]),
|
|
279
|
+
};
|
|
212
280
|
};
|
|
213
281
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
282
|
+
/**
|
|
283
|
+
* Input coverage of one host-scheduled fan-out plan: which required paths the
|
|
284
|
+
* bounded plan actually assigned complete evidence for. Capacity overflow and
|
|
285
|
+
* undiffable paths are both real gaps; the pipeline carries them so the check
|
|
286
|
+
* stays fail-closed until they are reviewed, removed, or explicitly ignored.
|
|
287
|
+
*/
|
|
288
|
+
export const fanOutInputCoverage = (input: {
|
|
289
|
+
readonly plan: ReviewUnitPlan;
|
|
290
|
+
readonly files: ReadonlyArray<ChangedFile>;
|
|
291
|
+
readonly totalFiles: number;
|
|
292
|
+
readonly anchorFiles: ReadonlyArray<ChangedFile>;
|
|
293
|
+
readonly totalAnchorFiles: number;
|
|
294
|
+
}): ReviewInputCoverage => {
|
|
295
|
+
const plan = input.plan;
|
|
219
296
|
const assignedPaths = sortedUnique(plan.units.flatMap((unit) => unit.paths));
|
|
220
|
-
const unassignedPaths = sortedUnique(
|
|
297
|
+
const unassignedPaths = sortedUnique(plan.unassignedPaths);
|
|
221
298
|
const reasons: Array<string> = [];
|
|
222
299
|
if (plan.truncated) {
|
|
223
|
-
reasons.push(
|
|
300
|
+
reasons.push(
|
|
301
|
+
`review range exposed ${input.files.length} of ${input.totalFiles} required files`,
|
|
302
|
+
);
|
|
224
303
|
}
|
|
225
304
|
if (plan.undiffablePaths.length > 0) {
|
|
226
305
|
reasons.push(
|
|
@@ -252,406 +331,23 @@ const fanOutInputCoverage = (
|
|
|
252
331
|
if (plan.unassignedPaths.length > 0) {
|
|
253
332
|
reasons.push(boundedListReason("fan-out capacity left paths unassigned", plan.unassignedPaths));
|
|
254
333
|
}
|
|
255
|
-
return
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
};
|
|
264
|
-
|
|
265
|
-
const sameStrings = (left: ReadonlyArray<string>, right: ReadonlyArray<string>): boolean =>
|
|
266
|
-
left.length === right.length && left.every((value, index) => value === right[index]);
|
|
267
|
-
|
|
268
|
-
const candidateKey = (candidate: ReviewCandidate): string =>
|
|
269
|
-
JSON.stringify(Schema.encodeSync(ReviewCandidate)(candidate));
|
|
270
|
-
|
|
271
|
-
const sameCandidates = (
|
|
272
|
-
left: ReadonlyArray<ReviewCandidate>,
|
|
273
|
-
right: ReadonlyArray<ReviewCandidate>,
|
|
274
|
-
): boolean =>
|
|
275
|
-
left.length === right.length &&
|
|
276
|
-
left.every((candidate, index) => {
|
|
277
|
-
const corresponding = right[index];
|
|
278
|
-
return corresponding !== undefined && candidateKey(candidate) === candidateKey(corresponding);
|
|
279
|
-
});
|
|
280
|
-
|
|
281
|
-
const delegationDeclarations = (trace: ToolTrace) =>
|
|
282
|
-
[...trace.declared].flatMap(([id, declaration]) => {
|
|
283
|
-
if (declaration.toolName !== "delegate_file_review") return [];
|
|
284
|
-
const request = Schema.decodeUnknownOption(FileReviewRequest)(declaration.parameters);
|
|
285
|
-
return Option.isNone(request) ? [] : [{ id, request: request.value }];
|
|
286
|
-
});
|
|
287
|
-
|
|
288
|
-
const failureTag = (trace: ToolTrace, id: string): string | undefined => {
|
|
289
|
-
const failed = trace.failed.get(id);
|
|
290
|
-
if (failed !== undefined) return failed.errorTag;
|
|
291
|
-
const succeeded = trace.succeeded.get(id);
|
|
292
|
-
if (succeeded === undefined) return undefined;
|
|
293
|
-
const returned = Schema.decodeUnknownOption(FileReviewDelegationFailure)(succeeded.result);
|
|
294
|
-
if (Option.isNone(returned)) return undefined;
|
|
295
|
-
return returned.value._tag === "FileReviewUnitFailed"
|
|
296
|
-
? `${returned.value._tag}:${returned.value.childErrorTag}`
|
|
297
|
-
: returned.value._tag;
|
|
298
|
-
};
|
|
299
|
-
|
|
300
|
-
const exactDiscoveryRequest = (request: FileReviewRequest, pass: ReviewDiscoveryPass): boolean =>
|
|
301
|
-
request.phase === "discovery" &&
|
|
302
|
-
request.workId === pass.passId &&
|
|
303
|
-
request.unitId === pass.unitId &&
|
|
304
|
-
request.perspective === pass.perspective &&
|
|
305
|
-
sameStrings(request.paths, pass.paths) &&
|
|
306
|
-
sameStrings(request.evidenceShardIds, pass.evidenceShardIds) &&
|
|
307
|
-
sameStrings(request.riskCategories, pass.riskCategories) &&
|
|
308
|
-
request.candidates.length === 0;
|
|
309
|
-
|
|
310
|
-
const validCandidate = (
|
|
311
|
-
candidate: ReviewCandidate,
|
|
312
|
-
pass: ReviewDiscoveryPass,
|
|
313
|
-
unit: ReviewUnit,
|
|
314
|
-
files: ReadonlyArray<ChangedFile>,
|
|
315
|
-
anchorFiles: ReadonlyArray<ChangedFile>,
|
|
316
|
-
): boolean => {
|
|
317
|
-
const allowed = new Set(pass.paths);
|
|
318
|
-
const kind = candidate._tag === "FindingCandidate" ? "finding" : "concern";
|
|
319
|
-
const idPrefix = `${pass.passId}:${kind}:`;
|
|
320
|
-
return (
|
|
321
|
-
candidate.candidateId.startsWith(idPrefix) &&
|
|
322
|
-
/^\d{3}$/.test(candidate.candidateId.slice(idPrefix.length)) &&
|
|
323
|
-
candidate.workId === pass.passId &&
|
|
324
|
-
candidate.unitId === pass.unitId &&
|
|
325
|
-
candidate.evidencePaths.length > 0 &&
|
|
326
|
-
candidate.evidencePaths.every((path) => allowed.has(path)) &&
|
|
327
|
-
(candidate._tag !== "FindingCandidate" ||
|
|
328
|
-
(allowed.has(candidate.finding.path) &&
|
|
329
|
-
anchorViolation(candidate.finding, anchorFiles) === undefined &&
|
|
330
|
-
findingAnchorInUnitEvidence(candidate.finding, unit, files)))
|
|
331
|
-
);
|
|
332
|
-
};
|
|
333
|
-
|
|
334
|
-
interface AssuranceAssessment {
|
|
335
|
-
readonly assurance: ReviewAssurance;
|
|
336
|
-
readonly confirmedFindings: ReadonlyArray<ReviewFinding>;
|
|
337
|
-
readonly confirmedConcerns: ReadonlyArray<ReviewConcern>;
|
|
338
|
-
readonly walkthrough: ReadonlyArray<WalkthroughEntry>;
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
const flatAssurance = (): AssuranceAssessment => ({
|
|
342
|
-
assurance: ReviewAssurance.make({
|
|
343
|
-
status: "unverified",
|
|
344
|
-
requiredGeneralDiscoveryPasses: 1,
|
|
345
|
-
completedGeneralDiscoveryPasses: 1,
|
|
346
|
-
requiredSpecialistPasses: 0,
|
|
347
|
-
completedSpecialistPasses: 0,
|
|
348
|
-
requiredVerificationPasses: 1,
|
|
349
|
-
completedVerificationPasses: 0,
|
|
350
|
-
discoveredCandidates: 0,
|
|
351
|
-
confirmedCandidates: 0,
|
|
352
|
-
rejectedCandidates: 0,
|
|
353
|
-
unsettledCandidates: 0,
|
|
354
|
-
failedPasses: [],
|
|
355
|
-
reasons: [
|
|
356
|
-
"flat review has no independent candidate-verification pass; use the fan-out pipeline for a settled assurance result",
|
|
357
|
-
],
|
|
358
|
-
}),
|
|
359
|
-
confirmedFindings: [],
|
|
360
|
-
confirmedConcerns: [],
|
|
361
|
-
walkthrough: [],
|
|
362
|
-
});
|
|
363
|
-
|
|
364
|
-
const fanOutAssurance = (
|
|
365
|
-
files: ReadonlyArray<ChangedFile>,
|
|
366
|
-
totalFiles: number,
|
|
367
|
-
anchorFiles: ReadonlyArray<ChangedFile>,
|
|
368
|
-
trace: ToolTrace,
|
|
369
|
-
): AssuranceAssessment => {
|
|
370
|
-
const plan = planReviewUnits(files, { totalChangedFiles: totalFiles });
|
|
371
|
-
const declarations = delegationDeclarations(trace);
|
|
372
|
-
const consumedDeclarationIds = new Set<string>();
|
|
373
|
-
const failedPasses: Array<FailedReviewPass> = [];
|
|
374
|
-
const reasons: Array<string> = [];
|
|
375
|
-
const candidatesByUnit = new Map<string, Array<ReviewCandidate>>();
|
|
376
|
-
const walkthrough: Array<WalkthroughEntry> = [];
|
|
377
|
-
let completedGeneralDiscoveryPasses = 0;
|
|
378
|
-
let completedSpecialistPasses = 0;
|
|
379
|
-
|
|
380
|
-
for (const pass of plan.discoveryPasses) {
|
|
381
|
-
const unit = plan.units.find((candidate) => candidate.unitId === pass.unitId);
|
|
382
|
-
const matching = declarations.filter(({ request }) => exactDiscoveryRequest(request, pass));
|
|
383
|
-
const stage = pass.perspective === "risk-specialist" ? "specialist" : "discovery";
|
|
384
|
-
if (matching.length !== 1) {
|
|
385
|
-
failedPasses.push(
|
|
386
|
-
FailedReviewPass.make({
|
|
387
|
-
workId: pass.passId,
|
|
388
|
-
stage,
|
|
389
|
-
errorTag: matching.length === 0 ? "PassNotAssigned" : "PassAssignedMultipleTimes",
|
|
390
|
-
}),
|
|
391
|
-
);
|
|
392
|
-
continue;
|
|
393
|
-
}
|
|
394
|
-
const call = matching[0];
|
|
395
|
-
if (call === undefined) {
|
|
396
|
-
failedPasses.push(
|
|
397
|
-
FailedReviewPass.make({
|
|
398
|
-
workId: pass.passId,
|
|
399
|
-
stage,
|
|
400
|
-
errorTag: "PassLookupInvariantFailed",
|
|
401
|
-
}),
|
|
402
|
-
);
|
|
403
|
-
continue;
|
|
404
|
-
}
|
|
405
|
-
consumedDeclarationIds.add(call.id);
|
|
406
|
-
const failure = failureTag(trace, call.id);
|
|
407
|
-
const succeeded = trace.succeeded.get(call.id);
|
|
408
|
-
const result =
|
|
409
|
-
succeeded === undefined
|
|
410
|
-
? Option.none()
|
|
411
|
-
: Schema.decodeUnknownOption(FileReviewUnitResult)(succeeded.result);
|
|
412
|
-
if (
|
|
413
|
-
failure !== undefined ||
|
|
414
|
-
Option.isNone(result) ||
|
|
415
|
-
result.value.phase !== "discovery" ||
|
|
416
|
-
result.value.workId !== pass.passId ||
|
|
417
|
-
result.value.unitId !== pass.unitId ||
|
|
418
|
-
result.value.assessments.length !== 0
|
|
419
|
-
) {
|
|
420
|
-
failedPasses.push(
|
|
421
|
-
FailedReviewPass.make({
|
|
422
|
-
workId: pass.passId,
|
|
423
|
-
stage,
|
|
424
|
-
errorTag: failure ?? "DiscoveryDidNotSettleExactly",
|
|
425
|
-
}),
|
|
426
|
-
);
|
|
427
|
-
continue;
|
|
428
|
-
}
|
|
429
|
-
const ids = new Set<string>();
|
|
430
|
-
let candidatesValid = true;
|
|
431
|
-
for (const candidate of result.value.candidates) {
|
|
432
|
-
if (
|
|
433
|
-
unit === undefined ||
|
|
434
|
-
ids.has(candidate.candidateId) ||
|
|
435
|
-
!validCandidate(candidate, pass, unit, files, anchorFiles)
|
|
436
|
-
) {
|
|
437
|
-
candidatesValid = false;
|
|
438
|
-
break;
|
|
439
|
-
}
|
|
440
|
-
ids.add(candidate.candidateId);
|
|
441
|
-
}
|
|
442
|
-
const unitCandidates = candidatesByUnit.get(pass.unitId) ?? [];
|
|
443
|
-
const unitIds = new Set(unitCandidates.map((candidate) => candidate.candidateId));
|
|
444
|
-
if (result.value.candidates.some((candidate) => unitIds.has(candidate.candidateId))) {
|
|
445
|
-
candidatesValid = false;
|
|
446
|
-
}
|
|
447
|
-
if (!candidatesValid) {
|
|
448
|
-
failedPasses.push(
|
|
449
|
-
FailedReviewPass.make({
|
|
450
|
-
workId: pass.passId,
|
|
451
|
-
stage,
|
|
452
|
-
errorTag: "DiscoveryCandidateMismatch",
|
|
453
|
-
}),
|
|
454
|
-
);
|
|
455
|
-
continue;
|
|
456
|
-
}
|
|
457
|
-
if (stage === "specialist") {
|
|
458
|
-
completedSpecialistPasses += 1;
|
|
459
|
-
} else {
|
|
460
|
-
completedGeneralDiscoveryPasses += 1;
|
|
461
|
-
}
|
|
462
|
-
const subjectKeys = new Set(unitCandidates.map(reviewCandidateSubjectKey));
|
|
463
|
-
for (const candidate of result.value.candidates) {
|
|
464
|
-
const subjectKey = reviewCandidateSubjectKey(candidate);
|
|
465
|
-
if (subjectKeys.has(subjectKey)) continue;
|
|
466
|
-
subjectKeys.add(subjectKey);
|
|
467
|
-
unitCandidates.push(candidate);
|
|
468
|
-
}
|
|
469
|
-
candidatesByUnit.set(pass.unitId, unitCandidates);
|
|
470
|
-
if (pass.perspective === "general") {
|
|
471
|
-
const allowed = new Set(pass.paths);
|
|
472
|
-
walkthrough.push(...result.value.fileSummaries.filter((entry) => allowed.has(entry.path)));
|
|
473
|
-
}
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
const confirmedCandidates: Array<ReviewCandidate> = [];
|
|
477
|
-
let rejectedCandidates = 0;
|
|
478
|
-
let unsettledCandidates = 0;
|
|
479
|
-
let requiredVerificationPasses = 0;
|
|
480
|
-
let completedVerificationPasses = 0;
|
|
481
|
-
for (const unit of plan.units) {
|
|
482
|
-
const candidates = candidatesByUnit.get(unit.unitId) ?? [];
|
|
483
|
-
if (candidates.length === 0) continue;
|
|
484
|
-
requiredVerificationPasses += 1;
|
|
485
|
-
const workId = `${unit.unitId}-verification`;
|
|
486
|
-
const matching = declarations.filter(
|
|
487
|
-
({ request }) =>
|
|
488
|
-
request.phase === "verification" &&
|
|
489
|
-
request.workId === workId &&
|
|
490
|
-
request.unitId === unit.unitId &&
|
|
491
|
-
request.perspective === "candidate-verification" &&
|
|
492
|
-
sameStrings(request.paths, unit.paths) &&
|
|
493
|
-
sameStrings(
|
|
494
|
-
request.evidenceShardIds,
|
|
495
|
-
unit.evidenceShards.map((shard) => shard.shardId),
|
|
496
|
-
) &&
|
|
497
|
-
sameStrings(request.riskCategories, unit.riskCategories) &&
|
|
498
|
-
sameCandidates(request.candidates, candidates),
|
|
499
|
-
);
|
|
500
|
-
if (matching.length !== 1) {
|
|
501
|
-
unsettledCandidates += candidates.length;
|
|
502
|
-
failedPasses.push(
|
|
503
|
-
FailedReviewPass.make({
|
|
504
|
-
workId,
|
|
505
|
-
stage: "verification",
|
|
506
|
-
errorTag:
|
|
507
|
-
matching.length === 0
|
|
508
|
-
? "VerificationNotAssignedOrCandidateMismatch"
|
|
509
|
-
: "VerificationAssignedMultipleTimes",
|
|
510
|
-
}),
|
|
511
|
-
);
|
|
512
|
-
continue;
|
|
513
|
-
}
|
|
514
|
-
const call = matching[0];
|
|
515
|
-
if (call === undefined) {
|
|
516
|
-
unsettledCandidates += candidates.length;
|
|
517
|
-
failedPasses.push(
|
|
518
|
-
FailedReviewPass.make({
|
|
519
|
-
workId,
|
|
520
|
-
stage: "verification",
|
|
521
|
-
errorTag: "PassLookupInvariantFailed",
|
|
522
|
-
}),
|
|
523
|
-
);
|
|
524
|
-
continue;
|
|
525
|
-
}
|
|
526
|
-
consumedDeclarationIds.add(call.id);
|
|
527
|
-
const failure = failureTag(trace, call.id);
|
|
528
|
-
const succeeded = trace.succeeded.get(call.id);
|
|
529
|
-
const result =
|
|
530
|
-
succeeded === undefined
|
|
531
|
-
? Option.none()
|
|
532
|
-
: Schema.decodeUnknownOption(FileReviewUnitResult)(succeeded.result);
|
|
533
|
-
if (
|
|
534
|
-
failure !== undefined ||
|
|
535
|
-
Option.isNone(result) ||
|
|
536
|
-
result.value.phase !== "verification" ||
|
|
537
|
-
result.value.workId !== workId ||
|
|
538
|
-
result.value.unitId !== unit.unitId ||
|
|
539
|
-
result.value.candidates.length !== 0 ||
|
|
540
|
-
result.value.fileSummaries.length !== 0
|
|
541
|
-
) {
|
|
542
|
-
unsettledCandidates += candidates.length;
|
|
543
|
-
failedPasses.push(
|
|
544
|
-
FailedReviewPass.make({
|
|
545
|
-
workId,
|
|
546
|
-
stage: "verification",
|
|
547
|
-
errorTag: failure ?? "VerificationDidNotSettleExactly",
|
|
548
|
-
}),
|
|
549
|
-
);
|
|
550
|
-
continue;
|
|
551
|
-
}
|
|
552
|
-
const expectedIds = new Set(candidates.map((candidate) => candidate.candidateId));
|
|
553
|
-
const assessedIds = new Set<string>();
|
|
554
|
-
const exactAssessments = result.value.assessments.every((assessment) => {
|
|
555
|
-
if (!expectedIds.has(assessment.candidateId) || assessedIds.has(assessment.candidateId)) {
|
|
556
|
-
return false;
|
|
557
|
-
}
|
|
558
|
-
assessedIds.add(assessment.candidateId);
|
|
559
|
-
return true;
|
|
560
|
-
});
|
|
561
|
-
if (
|
|
562
|
-
expectedIds.size !== candidates.length ||
|
|
563
|
-
!exactAssessments ||
|
|
564
|
-
assessedIds.size !== expectedIds.size
|
|
565
|
-
) {
|
|
566
|
-
unsettledCandidates += candidates.length;
|
|
567
|
-
failedPasses.push(
|
|
568
|
-
FailedReviewPass.make({
|
|
569
|
-
workId,
|
|
570
|
-
stage: "verification",
|
|
571
|
-
errorTag: "VerificationAssessmentMismatch",
|
|
572
|
-
}),
|
|
573
|
-
);
|
|
574
|
-
continue;
|
|
575
|
-
}
|
|
576
|
-
completedVerificationPasses += 1;
|
|
577
|
-
const byId = new Map(
|
|
578
|
-
candidates.map((candidate) => [candidate.candidateId, candidate] as const),
|
|
579
|
-
);
|
|
580
|
-
for (const assessment of result.value.assessments) {
|
|
581
|
-
if (assessment.disposition === "confirmed") {
|
|
582
|
-
const candidate = byId.get(assessment.candidateId);
|
|
583
|
-
if (candidate !== undefined) confirmedCandidates.push(candidate);
|
|
584
|
-
} else {
|
|
585
|
-
rejectedCandidates += 1;
|
|
586
|
-
}
|
|
587
|
-
}
|
|
588
|
-
}
|
|
589
|
-
|
|
590
|
-
const unexpected = [...trace.declared].filter(
|
|
591
|
-
([id, declaration]) =>
|
|
592
|
-
declaration.toolName === "delegate_file_review" && !consumedDeclarationIds.has(id),
|
|
593
|
-
);
|
|
594
|
-
for (const [, declaration] of unexpected) {
|
|
595
|
-
const request = Schema.decodeUnknownOption(FileReviewRequest)(declaration.parameters);
|
|
596
|
-
failedPasses.push(
|
|
597
|
-
FailedReviewPass.make({
|
|
598
|
-
workId: Option.isSome(request) ? request.value.workId : "invalid-delegation-request",
|
|
599
|
-
stage:
|
|
600
|
-
Option.isSome(request) && request.value.phase === "verification"
|
|
601
|
-
? "verification"
|
|
602
|
-
: "discovery",
|
|
603
|
-
errorTag: "UnexpectedPass",
|
|
604
|
-
}),
|
|
605
|
-
);
|
|
606
|
-
}
|
|
607
|
-
if (failedPasses.length > 0) {
|
|
608
|
-
reasons.push(
|
|
609
|
-
boundedListReason(
|
|
610
|
-
"configured review passes did not settle",
|
|
611
|
-
failedPasses.map((pass) => `${pass.workId} (${pass.errorTag})`),
|
|
612
|
-
),
|
|
613
|
-
);
|
|
614
|
-
}
|
|
615
|
-
if (unsettledCandidates > 0) {
|
|
616
|
-
reasons.push(
|
|
617
|
-
`${unsettledCandidates} discovered candidate(s) did not receive exact verification`,
|
|
618
|
-
);
|
|
619
|
-
}
|
|
620
|
-
const requiredSpecialistPasses = plan.discoveryPasses.filter(
|
|
621
|
-
(pass) => pass.perspective === "risk-specialist",
|
|
622
|
-
).length;
|
|
623
|
-
const requiredGeneralDiscoveryPasses = plan.discoveryPasses.length - requiredSpecialistPasses;
|
|
624
|
-
const discoveredCandidates = [...candidatesByUnit.values()].reduce(
|
|
625
|
-
(total, candidates) => total + candidates.length,
|
|
626
|
-
0,
|
|
627
|
-
);
|
|
628
|
-
return {
|
|
629
|
-
assurance: ReviewAssurance.make({
|
|
630
|
-
status: reasons.length === 0 ? "settled" : "incomplete",
|
|
631
|
-
requiredGeneralDiscoveryPasses,
|
|
632
|
-
completedGeneralDiscoveryPasses,
|
|
633
|
-
requiredSpecialistPasses,
|
|
634
|
-
completedSpecialistPasses,
|
|
635
|
-
requiredVerificationPasses,
|
|
636
|
-
completedVerificationPasses,
|
|
637
|
-
discoveredCandidates,
|
|
638
|
-
confirmedCandidates: confirmedCandidates.length,
|
|
639
|
-
rejectedCandidates,
|
|
640
|
-
unsettledCandidates,
|
|
641
|
-
failedPasses,
|
|
334
|
+
return anchorSurfaceAdjusted(
|
|
335
|
+
ReviewInputCoverage.make({
|
|
336
|
+
status: reasons.length === 0 ? "complete" : "incomplete",
|
|
337
|
+
requiredPaths: sortedUnique(input.files.map((file) => file.path)),
|
|
338
|
+
assignedPaths,
|
|
339
|
+
partialPaths: plan.partialEvidencePaths,
|
|
340
|
+
unassignedPaths,
|
|
341
|
+
undiffablePaths: sortedUnique(plan.undiffablePaths),
|
|
642
342
|
reasons,
|
|
643
343
|
}),
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
confirmedConcerns: confirmedCandidates.flatMap((candidate) =>
|
|
648
|
-
candidate._tag === "ConcernCandidate" ? [candidate.concern] : [],
|
|
649
|
-
),
|
|
650
|
-
walkthrough,
|
|
651
|
-
};
|
|
344
|
+
input.anchorFiles,
|
|
345
|
+
input.totalAnchorFiles,
|
|
346
|
+
);
|
|
652
347
|
};
|
|
653
348
|
|
|
654
|
-
|
|
349
|
+
/** Compatibility aggregate over the two precise claims. */
|
|
350
|
+
export const compatibilityCoverage = (
|
|
655
351
|
inputCoverage: ReviewInputCoverage,
|
|
656
352
|
assurance: ReviewAssurance,
|
|
657
353
|
): ReviewCoverage => {
|
|
@@ -678,84 +374,3 @@ const compatibilityCoverage = (
|
|
|
678
374
|
reasons: [...inputCoverage.reasons, ...(assuranceIncomplete ? assurance.reasons : [])],
|
|
679
375
|
});
|
|
680
376
|
};
|
|
681
|
-
|
|
682
|
-
export interface ReviewPipelineAssessment {
|
|
683
|
-
readonly inputCoverage: ReviewInputCoverage;
|
|
684
|
-
readonly assurance: ReviewAssurance;
|
|
685
|
-
/** Deprecated compatibility aggregate. */
|
|
686
|
-
readonly coverage: ReviewCoverage;
|
|
687
|
-
readonly confirmedFindings: ReadonlyArray<ReviewFinding>;
|
|
688
|
-
readonly confirmedConcerns: ReadonlyArray<ReviewConcern>;
|
|
689
|
-
readonly walkthrough: ReadonlyArray<WalkthroughEntry>;
|
|
690
|
-
}
|
|
691
|
-
|
|
692
|
-
/** Assess one settled run without trusting coordinator prose or findings. */
|
|
693
|
-
export const assessReviewPipeline = (input: {
|
|
694
|
-
readonly shape: ReviewShape;
|
|
695
|
-
readonly files: ReadonlyArray<ChangedFile>;
|
|
696
|
-
readonly totalFiles: number;
|
|
697
|
-
readonly anchorFiles: ReadonlyArray<ChangedFile>;
|
|
698
|
-
readonly totalAnchorFiles: number;
|
|
699
|
-
readonly events: ReadonlyArray<RunEvent>;
|
|
700
|
-
}): ReviewPipelineAssessment => {
|
|
701
|
-
const trace = toolTrace(input.events);
|
|
702
|
-
let inputCoverage =
|
|
703
|
-
input.shape === "fan-out"
|
|
704
|
-
? fanOutInputCoverage(input.files, input.totalFiles)
|
|
705
|
-
: flatInputCoverage(input.files, input.totalFiles, trace);
|
|
706
|
-
if (input.anchorFiles.length < input.totalAnchorFiles) {
|
|
707
|
-
inputCoverage = ReviewInputCoverage.make({
|
|
708
|
-
...inputCoverage,
|
|
709
|
-
status: "incomplete",
|
|
710
|
-
reasons: [
|
|
711
|
-
...inputCoverage.reasons,
|
|
712
|
-
`full pull-request anchor surface exposed ${input.anchorFiles.length} of ${input.totalAnchorFiles} required files`,
|
|
713
|
-
],
|
|
714
|
-
});
|
|
715
|
-
}
|
|
716
|
-
const assessed =
|
|
717
|
-
input.shape === "fan-out"
|
|
718
|
-
? fanOutAssurance(input.files, input.totalFiles, input.anchorFiles, trace)
|
|
719
|
-
: flatAssurance();
|
|
720
|
-
return {
|
|
721
|
-
inputCoverage,
|
|
722
|
-
assurance: assessed.assurance,
|
|
723
|
-
coverage: compatibilityCoverage(inputCoverage, assessed.assurance),
|
|
724
|
-
confirmedFindings: assessed.confirmedFindings,
|
|
725
|
-
confirmedConcerns: assessed.confirmedConcerns,
|
|
726
|
-
walkthrough: assessed.walkthrough,
|
|
727
|
-
};
|
|
728
|
-
};
|
|
729
|
-
|
|
730
|
-
/** Compatibility helper; prefer assessReviewPipeline for precise claims. */
|
|
731
|
-
export const assessReviewCoverage = (input: {
|
|
732
|
-
readonly shape: ReviewShape;
|
|
733
|
-
readonly files: ReadonlyArray<ChangedFile>;
|
|
734
|
-
readonly totalFiles: number;
|
|
735
|
-
readonly anchorFiles: ReadonlyArray<ChangedFile>;
|
|
736
|
-
readonly totalAnchorFiles: number;
|
|
737
|
-
readonly events: ReadonlyArray<RunEvent>;
|
|
738
|
-
}): ReviewCoverage => assessReviewPipeline(input).coverage;
|
|
739
|
-
|
|
740
|
-
/** Host-verified summaries from successful general discovery passes only. */
|
|
741
|
-
export const collectUnitFileSummaries = (
|
|
742
|
-
events: ReadonlyArray<RunEvent>,
|
|
743
|
-
): ReadonlyArray<WalkthroughEntry> => {
|
|
744
|
-
const trace = toolTrace(events);
|
|
745
|
-
return delegationDeclarations(trace).flatMap(({ id, request }) => {
|
|
746
|
-
if (request.phase !== "discovery" || request.perspective !== "general") return [];
|
|
747
|
-
const success = trace.succeeded.get(id);
|
|
748
|
-
if (success === undefined || trace.failed.has(id)) return [];
|
|
749
|
-
const result = Schema.decodeUnknownOption(FileReviewUnitResult)(success.result);
|
|
750
|
-
if (
|
|
751
|
-
Option.isNone(result) ||
|
|
752
|
-
result.value.phase !== "discovery" ||
|
|
753
|
-
result.value.workId !== request.workId ||
|
|
754
|
-
result.value.unitId !== request.unitId
|
|
755
|
-
) {
|
|
756
|
-
return [];
|
|
757
|
-
}
|
|
758
|
-
const assigned = new Set(request.paths);
|
|
759
|
-
return result.value.fileSummaries.filter((entry) => assigned.has(entry.path));
|
|
760
|
-
});
|
|
761
|
-
};
|