@openai/codex-security 0.1.23 → 0.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -23
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/scan-contract.md +10 -0
- package/_bundled_plugin/scripts/deep_scan_workbench.py +6 -1
- package/_bundled_plugin/scripts/finalize_scan_contract.py +27 -78
- package/_bundled_plugin/scripts/generate_in_scope_files.py +2 -8
- package/_bundled_plugin/scripts/generate_rank_input.py +1 -5
- package/_bundled_plugin/scripts/rank_preview.py +12 -1
- package/_bundled_plugin/scripts/report_projection.py +8 -14
- package/_bundled_plugin/scripts/validate_scan_contract.py +1 -3
- package/_bundled_plugin/scripts/windows_scan_local_files.py +5 -15
- package/_bundled_plugin/scripts/workbench_cli.py +14 -1
- package/_bundled_plugin/scripts/workbench_dashboard.py +41 -18
- package/_bundled_plugin/scripts/workbench_db.py +127 -541
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +31 -13
- package/_bundled_plugin/scripts/workbench_findings.py +8 -3
- package/_bundled_plugin/scripts/workbench_native_indexes.py +1 -2
- package/_bundled_plugin/scripts/workbench_publication.py +520 -0
- package/_bundled_plugin/scripts/workbench_remediation.py +37 -0
- package/_bundled_plugin/scripts/workbench_saved_results.py +298 -90
- package/_bundled_plugin/scripts/workbench_scan_history.py +375 -68
- package/_bundled_plugin/scripts/workbench_scan_usage.py +9 -33
- package/_bundled_plugin/scripts/workbench_schema.py +49 -21
- package/_bundled_plugin/scripts/workbench_target.py +96 -17
- package/_bundled_plugin/scripts/workbench_validation.py +1 -5
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +1 -5
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +6 -1
- package/dist/api.d.ts +18 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +350 -28
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +35 -3
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +118 -35
- package/dist/cli.js.map +1 -1
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +4 -2
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +21 -6
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +46 -2
- package/dist/config.js.map +1 -1
- package/dist/cost.d.ts +1 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +23 -11
- package/dist/cost.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +2 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts +3 -0
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +122 -36
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +4 -3
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +18 -9
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +2 -7
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +46 -24
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.d.ts.map +1 -1
- package/dist/deduplication/deduplication.js +186 -23
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/scan.d.ts +9 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +24 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-progress.d.ts +21 -0
- package/dist/deep-progress.d.ts.map +1 -0
- package/dist/deep-progress.js +108 -0
- package/dist/deep-progress.js.map +1 -0
- package/dist/errors.d.ts +13 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +7 -0
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/mock-scan.d.ts +4 -0
- package/dist/mock-scan.d.ts.map +1 -0
- package/dist/mock-scan.js +210 -0
- package/dist/mock-scan.js.map +1 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +2 -2
- package/dist/publish.js.map +1 -1
- package/dist/runtime.d.ts +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +54 -14
- package/dist/runtime.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +12 -2
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +11 -2
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +237 -39
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +3 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +70 -3
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +21 -0
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/server/api.d.ts +6 -0
- package/dist/server/api.d.ts.map +1 -0
- package/dist/server/api.js +4 -0
- package/dist/server/api.js.map +1 -0
- package/dist/server/dashboard/app.css +1 -1
- package/dist/server/embeddings.d.ts +2 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +8 -3
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/index.js +2 -28
- package/dist/server/index.js.map +1 -1
- package/dist/server/serve.d.ts +2 -0
- package/dist/server/serve.d.ts.map +1 -0
- package/dist/server/serve.js +28 -0
- package/dist/server/serve.js.map +1 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +3 -5
- package/dist/targets.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +10 -5
|
@@ -13,18 +13,12 @@ declare const reviewSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
13
13
|
mergedFinding: z.ZodOptional<z.ZodNull>;
|
|
14
14
|
}, z.core.$strip>], "decision">;
|
|
15
15
|
declare const screeningSchema: z.ZodObject<{
|
|
16
|
-
decisions: z.
|
|
16
|
+
decisions: z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
17
17
|
decision: z.ZodLiteral<"SAME">;
|
|
18
18
|
rationale: z.ZodString;
|
|
19
|
-
canonicalFindingId: z.ZodString;
|
|
20
|
-
mergedFinding: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
21
|
-
findingIds: z.ZodTuple<[z.ZodString, z.ZodString], null>;
|
|
22
19
|
}, z.core.$strict>, z.ZodObject<{
|
|
23
20
|
decision: z.ZodLiteral<"DISTINCT">;
|
|
24
21
|
rationale: z.ZodString;
|
|
25
|
-
canonicalFindingId: z.ZodOptional<z.ZodNull>;
|
|
26
|
-
mergedFinding: z.ZodOptional<z.ZodNull>;
|
|
27
|
-
findingIds: z.ZodTuple<[z.ZodString, z.ZodString], null>;
|
|
28
22
|
}, z.core.$strict>], "decision">>;
|
|
29
23
|
}, z.core.$strict>;
|
|
30
24
|
export type ScreeningResult = z.infer<typeof screeningSchema>;
|
|
@@ -34,6 +28,7 @@ export interface DeduplicationReviewer {
|
|
|
34
28
|
reviewPair(findings: readonly Finding[]): Promise<DuplicateDecision>;
|
|
35
29
|
}
|
|
36
30
|
export declare function pairKey(ids: readonly string[]): string;
|
|
31
|
+
export declare function screeningPairSlot(index: number): string;
|
|
37
32
|
export declare function validateReview(value: unknown, findings: readonly Finding[]): DuplicateDecision;
|
|
38
33
|
export declare function validateScreening(value: unknown, findings: readonly Finding[]): ScreeningResult;
|
|
39
34
|
export declare class CodexDeduplicationReviewer implements DeduplicationReviewer {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deduplication-reviewer.d.ts","sourceRoot":"","sources":["../../src/deduplication/deduplication-reviewer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,OAAO,CAAC;AAG1B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAgB3D,QAAA,MAAM,YAAY;;;;;;;;;;+BAGhB,CAAC;
|
|
1
|
+
{"version":3,"file":"deduplication-reviewer.d.ts","sourceRoot":"","sources":["../../src/deduplication/deduplication-reviewer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,OAAO,CAAC;AAG1B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAgB3D,QAAA,MAAM,YAAY;;;;;;;;;;+BAGhB,CAAC;AAcH,QAAA,MAAM,eAAe;;;;;;;;kBAIV,CAAC;AA6BZ,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAC9D,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAE7D,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC/D,UAAU,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CACtE;AAED,wBAAgB,OAAO,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAEtD;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED,wBAAgB,cAAc,CAC5B,KAAK,EAAE,OAAO,EACd,QAAQ,EAAE,SAAS,OAAO,EAAE,GAC3B,iBAAiB,CAYnB;AAED,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,OAAO,EACd,QAAQ,EAAE,SAAS,OAAO,EAAE,GAC3B,eAAe,CAajB;AA0BD,qBAAa,0BAA2B,YAAW,qBAAqB;IAC1D,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC;IAE7D,MAAM,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC;IAW9D,UAAU,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAC;CAa3E"}
|
|
@@ -19,13 +19,22 @@ const reviewSchema = z.discriminatedUnion("decision", [
|
|
|
19
19
|
sameSchema,
|
|
20
20
|
distinctSchema,
|
|
21
21
|
]);
|
|
22
|
-
const
|
|
22
|
+
const screeningSameSchema = z.object({
|
|
23
|
+
decision: z.literal("SAME"),
|
|
24
|
+
rationale,
|
|
25
|
+
});
|
|
26
|
+
const screeningDistinctSchema = z.object({
|
|
27
|
+
decision: z.literal("DISTINCT"),
|
|
28
|
+
rationale,
|
|
29
|
+
});
|
|
30
|
+
const screeningDecisionSchema = z.discriminatedUnion("decision", [
|
|
31
|
+
screeningSameSchema.strict(),
|
|
32
|
+
screeningDistinctSchema.strict(),
|
|
33
|
+
]);
|
|
34
|
+
// The host assigns exact slot names; finding IDs stay out of model output.
|
|
23
35
|
const screeningSchema = z
|
|
24
36
|
.object({
|
|
25
|
-
decisions: z.
|
|
26
|
-
sameSchema.extend({ findingIds }).strict(),
|
|
27
|
-
distinctSchema.extend({ findingIds }).strict(),
|
|
28
|
-
])),
|
|
37
|
+
decisions: z.record(z.string(), screeningDecisionSchema),
|
|
29
38
|
})
|
|
30
39
|
.strict();
|
|
31
40
|
let validateMergedFinding;
|
|
@@ -43,6 +52,9 @@ function requireMergedFinding(result) {
|
|
|
43
52
|
export function pairKey(ids) {
|
|
44
53
|
return JSON.stringify([...ids].sort());
|
|
45
54
|
}
|
|
55
|
+
export function screeningPairSlot(index) {
|
|
56
|
+
return `pair-${index + 1}`;
|
|
57
|
+
}
|
|
46
58
|
export function validateReview(value, findings) {
|
|
47
59
|
const result = reviewSchema.parse(value);
|
|
48
60
|
requireMergedFinding(result);
|
|
@@ -54,27 +66,35 @@ export function validateReview(value, findings) {
|
|
|
54
66
|
}
|
|
55
67
|
export function validateScreening(value, findings) {
|
|
56
68
|
const result = screeningSchema.parse(value);
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if (pair[0] === pair[1] || !required.has(key) || seen.has(key)) {
|
|
65
|
-
throw new Error("Submit each assigned anchor-neighbor pair exactly once.");
|
|
66
|
-
}
|
|
67
|
-
if (recommendation.decision === "SAME" &&
|
|
68
|
-
!pair.includes(recommendation.canonicalFindingId)) {
|
|
69
|
-
throw new Error("The canonical finding must belong to its assigned pair.");
|
|
70
|
-
}
|
|
71
|
-
seen.add(key);
|
|
72
|
-
}
|
|
73
|
-
if ([...required].some((key) => !seen.has(key))) {
|
|
74
|
-
throw new Error("Submit a decision for every assigned anchor-neighbor pair.");
|
|
69
|
+
const required = findings
|
|
70
|
+
.slice(1)
|
|
71
|
+
.map((_finding, index) => screeningPairSlot(index));
|
|
72
|
+
const submitted = Object.keys(result.decisions);
|
|
73
|
+
if (submitted.length !== required.length ||
|
|
74
|
+
required.some((slot) => !Object.hasOwn(result.decisions, slot))) {
|
|
75
|
+
throw new Error("Submit exactly the assigned screening pair slots.");
|
|
75
76
|
}
|
|
76
77
|
return result;
|
|
77
78
|
}
|
|
79
|
+
function screeningToolSchema(neighborCount) {
|
|
80
|
+
const decisionSchema = z.toJSONSchema(screeningDecisionSchema, {
|
|
81
|
+
target: "openapi-3.0",
|
|
82
|
+
});
|
|
83
|
+
const slots = Array.from({ length: neighborCount }, (_value, index) => screeningPairSlot(index));
|
|
84
|
+
return {
|
|
85
|
+
type: "object",
|
|
86
|
+
properties: {
|
|
87
|
+
decisions: {
|
|
88
|
+
type: "object",
|
|
89
|
+
properties: Object.fromEntries(slots.map((slot) => [slot, decisionSchema])),
|
|
90
|
+
required: slots,
|
|
91
|
+
additionalProperties: false,
|
|
92
|
+
},
|
|
93
|
+
},
|
|
94
|
+
required: ["decisions"],
|
|
95
|
+
additionalProperties: false,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
78
98
|
export class CodexDeduplicationReviewer {
|
|
79
99
|
runner;
|
|
80
100
|
constructor(runner) {
|
|
@@ -82,15 +102,17 @@ export class CodexDeduplicationReviewer {
|
|
|
82
102
|
}
|
|
83
103
|
async screen(findings) {
|
|
84
104
|
return await this.runner.run({
|
|
105
|
+
stage: "screening",
|
|
85
106
|
model: "gpt-5.6-luna",
|
|
86
107
|
effort: "xhigh",
|
|
87
108
|
prompt: screeningPrompt(findings),
|
|
88
|
-
schema:
|
|
109
|
+
schema: screeningToolSchema(findings.length - 1),
|
|
89
110
|
validate: (value) => validateScreening(value, findings),
|
|
90
111
|
});
|
|
91
112
|
}
|
|
92
113
|
async reviewPair(findings) {
|
|
93
114
|
return await this.runner.run({
|
|
115
|
+
stage: "pair-review",
|
|
94
116
|
model: "gpt-5.6-sol",
|
|
95
117
|
effort: "xhigh",
|
|
96
118
|
prompt: pairReviewPrompt(findings),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deduplication-reviewer.js","sourceRoot":"","sources":["../../src/deduplication/deduplication-reviewer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,OAAkC,MAAM,kBAAkB,CAAC;AAGlE,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAE/E,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACxE,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAC3B,SAAS;IACT,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;IAC9B,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;CACjD,CAAC,CAAC;AACH,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;IAC/B,SAAS;IACT,kBAAkB,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE;IACvC,aAAa,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE;CACnC,CAAC,CAAC;AACH,MAAM,YAAY,GAAG,CAAC,CAAC,kBAAkB,CAAC,UAAU,EAAE;IACpD,UAAU;IACV,cAAc;CACf,CAAC,CAAC;AACH,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACrD,MAAM,eAAe,GAAG,CAAC;KACtB,MAAM,CAAC;IACN,SAAS,EAAE,CAAC,CAAC,KAAK,CAChB,CAAC,CAAC,kBAAkB,CAAC,UAAU,EAAE;QAC/B,UAAU,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QAC1C,cAAc,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;KAC/C,CAAC,CACH;CACF,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,IAAI,qBAA4D,CAAC;AAEjE,SAAS,oBAAoB,CAAC,MAAyB;IACrD,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM;QAAE,OAAO;IACvC,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CACvB,YAAY,CACV,IAAI,GAAG,CACL,oDAAoD,EACpD,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB,EACD,MAAM,CACP,CACF,CAAC;QACF,qBAAqB,GAAG,IAAI,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,CAC5D,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CACjC,CAAC;IACJ,CAAC;IACD,IACE,CAAC,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC;QAC5C,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,KAAK,MAAM,CAAC,kBAAkB;QAE/D,MAAM,IAAI,KAAK,CACb,qHAAqH,CACtH,CAAC;AACN,CAAC;AAUD,MAAM,UAAU,OAAO,CAAC,GAAsB;IAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,KAAc,EACd,QAA4B;IAE5B,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAC7B,IACE,MAAM,CAAC,QAAQ,KAAK,MAAM;QAC1B,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,KAAK,MAAM,CAAC,kBAAkB,CAAC,EAC5E,CAAC;QACD,MAAM,IAAI,KAAK,CACb,6DAA6D,CAC9D,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,KAAc,EACd,QAA4B;IAE5B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC;IACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CACzE,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,cAAc,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QAC9C,oBAAoB,CAAC,cAAc,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,CAAC;QACvC,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CACb,yDAAyD,CAC1D,CAAC;QACJ,CAAC;QACD,IACE,cAAc,CAAC,QAAQ,KAAK,MAAM;YAClC,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,kBAAkB,CAAC,EACjD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,yDAAyD,CAC1D,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,CAAC;IACD,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,0BAA0B;IACR;IAA7B,YAA6B,MAAsC;QAAtC,WAAM,GAAN,MAAM,CAAgC;IAAG,CAAC;IAEvE,KAAK,CAAC,MAAM,CAAC,QAA4B;QACvC,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3B,KAAK,EAAE,cAAc;YACrB,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC;YACjC,MAAM,EAAE,CAAC,CAAC,YAAY,CAAC,eAAe,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;YAClE,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,QAAQ,CAAC;SACxD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,QAA4B;QAC3C,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3B,KAAK,EAAE,aAAa;YACpB,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC;YAClC,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,GAAG,CAAC,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;aAC3D;YACD,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC;SACrD,CAAC,CAAC;IACL,CAAC;CACF","sourcesContent":["import { z } from \"incur\";\nimport { readFileSync } from \"node:fs\";\nimport Ajv2020, { type ValidateFunction } from \"ajv/dist/2020.js\";\nimport type { Finding } from \"../models.js\";\nimport type { CodexReviewRunner } from \"./codex-review.js\";\nimport { pairReviewPrompt, screeningPrompt } from \"./deduplication-prompts.js\";\n\nconst rationale = z.string().refine((value) => value.trim().length > 0);\nconst sameSchema = z.object({\n decision: z.literal(\"SAME\"),\n rationale,\n canonicalFindingId: z.string(),\n mergedFinding: z.record(z.string(), z.unknown()),\n});\nconst distinctSchema = z.object({\n decision: z.literal(\"DISTINCT\"),\n rationale,\n canonicalFindingId: z.null().optional(),\n mergedFinding: z.null().optional(),\n});\nconst reviewSchema = z.discriminatedUnion(\"decision\", [\n sameSchema,\n distinctSchema,\n]);\nconst findingIds = z.tuple([z.string(), z.string()]);\nconst screeningSchema = z\n .object({\n decisions: z.array(\n z.discriminatedUnion(\"decision\", [\n sameSchema.extend({ findingIds }).strict(),\n distinctSchema.extend({ findingIds }).strict(),\n ]),\n ),\n })\n .strict();\n\nlet validateMergedFinding: ValidateFunction<Finding> | undefined;\n\nfunction requireMergedFinding(result: DuplicateDecision): void {\n if (result.decision !== \"SAME\") return;\n if (validateMergedFinding === undefined) {\n const schema = JSON.parse(\n readFileSync(\n new URL(\n \"../../_bundled_plugin/schemas/findings.schema.json\",\n import.meta.url,\n ),\n \"utf8\",\n ),\n );\n validateMergedFinding = new Ajv2020({ strict: false }).compile<Finding>(\n schema.properties.findings.items,\n );\n }\n if (\n !validateMergedFinding(result.mergedFinding) ||\n result.mergedFinding[\"findingId\"] !== result.canonicalFindingId\n )\n throw new Error(\n \"Every SAME decision requires a generated mergedFinding in the Finding schema with the canonical finding's identity.\",\n );\n}\n\nexport type ScreeningResult = z.infer<typeof screeningSchema>;\nexport type DuplicateDecision = z.infer<typeof reviewSchema>;\n\nexport interface DeduplicationReviewer {\n screen(findings: readonly Finding[]): Promise<ScreeningResult>;\n reviewPair(findings: readonly Finding[]): Promise<DuplicateDecision>;\n}\n\nexport function pairKey(ids: readonly string[]): string {\n return JSON.stringify([...ids].sort());\n}\n\nexport function validateReview(\n value: unknown,\n findings: readonly Finding[],\n): DuplicateDecision {\n const result = reviewSchema.parse(value);\n requireMergedFinding(result);\n if (\n result.decision === \"SAME\" &&\n !findings.some((finding) => finding.findingId === result.canonicalFindingId)\n ) {\n throw new Error(\n \"The canonical finding must belong to the assigned findings.\",\n );\n }\n return result;\n}\n\nexport function validateScreening(\n value: unknown,\n findings: readonly Finding[],\n): ScreeningResult {\n const result = screeningSchema.parse(value);\n const anchor = findings[0]!.findingId;\n const required = new Set(\n findings.slice(1).map((finding) => pairKey([anchor, finding.findingId])),\n );\n const seen = new Set<string>();\n for (const recommendation of result.decisions) {\n requireMergedFinding(recommendation);\n const pair = recommendation.findingIds;\n const key = pairKey(pair);\n if (pair[0] === pair[1] || !required.has(key) || seen.has(key)) {\n throw new Error(\n \"Submit each assigned anchor-neighbor pair exactly once.\",\n );\n }\n if (\n recommendation.decision === \"SAME\" &&\n !pair.includes(recommendation.canonicalFindingId)\n ) {\n throw new Error(\n \"The canonical finding must belong to its assigned pair.\",\n );\n }\n seen.add(key);\n }\n if ([...required].some((key) => !seen.has(key))) {\n throw new Error(\n \"Submit a decision for every assigned anchor-neighbor pair.\",\n );\n }\n return result;\n}\n\nexport class CodexDeduplicationReviewer implements DeduplicationReviewer {\n constructor(private readonly runner: Pick<CodexReviewRunner, \"run\">) {}\n\n async screen(findings: readonly Finding[]): Promise<ScreeningResult> {\n return await this.runner.run({\n model: \"gpt-5.6-luna\",\n effort: \"xhigh\",\n prompt: screeningPrompt(findings),\n schema: z.toJSONSchema(screeningSchema, { target: \"openapi-3.0\" }),\n validate: (value) => validateScreening(value, findings),\n });\n }\n\n async reviewPair(findings: readonly Finding[]): Promise<DuplicateDecision> {\n return await this.runner.run({\n model: \"gpt-5.6-sol\",\n effort: \"xhigh\",\n prompt: pairReviewPrompt(findings),\n schema: {\n type: \"object\",\n ...z.toJSONSchema(reviewSchema, { target: \"openapi-3.0\" }),\n },\n validate: (value) => validateReview(value, findings),\n });\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"deduplication-reviewer.js","sourceRoot":"","sources":["../../src/deduplication/deduplication-reviewer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,OAAkC,MAAM,kBAAkB,CAAC;AAGlE,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAE/E,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACxE,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAC3B,SAAS;IACT,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;IAC9B,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;CACjD,CAAC,CAAC;AACH,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;IAC/B,SAAS;IACT,kBAAkB,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE;IACvC,aAAa,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE;CACnC,CAAC,CAAC;AACH,MAAM,YAAY,GAAG,CAAC,CAAC,kBAAkB,CAAC,UAAU,EAAE;IACpD,UAAU;IACV,cAAc;CACf,CAAC,CAAC;AACH,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAC3B,SAAS;CACV,CAAC,CAAC;AACH,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;IAC/B,SAAS;CACV,CAAC,CAAC;AACH,MAAM,uBAAuB,GAAG,CAAC,CAAC,kBAAkB,CAAC,UAAU,EAAE;IAC/D,mBAAmB,CAAC,MAAM,EAAE;IAC5B,uBAAuB,CAAC,MAAM,EAAE;CACjC,CAAC,CAAC;AACH,2EAA2E;AAC3E,MAAM,eAAe,GAAG,CAAC;KACtB,MAAM,CAAC;IACN,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,uBAAuB,CAAC;CACzD,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,IAAI,qBAA4D,CAAC;AAEjE,SAAS,oBAAoB,CAAC,MAAyB;IACrD,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM;QAAE,OAAO;IACvC,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CACvB,YAAY,CACV,IAAI,GAAG,CACL,oDAAoD,EACpD,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB,EACD,MAAM,CACP,CACF,CAAC;QACF,qBAAqB,GAAG,IAAI,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,CAC5D,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CACjC,CAAC;IACJ,CAAC;IACD,IACE,CAAC,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC;QAC5C,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,KAAK,MAAM,CAAC,kBAAkB;QAE/D,MAAM,IAAI,KAAK,CACb,qHAAqH,CACtH,CAAC;AACN,CAAC;AAUD,MAAM,UAAU,OAAO,CAAC,GAAsB;IAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAa;IAC7C,OAAO,QAAQ,KAAK,GAAG,CAAC,EAAE,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,KAAc,EACd,QAA4B;IAE5B,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAC7B,IACE,MAAM,CAAC,QAAQ,KAAK,MAAM;QAC1B,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,KAAK,MAAM,CAAC,kBAAkB,CAAC,EAC5E,CAAC;QACD,MAAM,IAAI,KAAK,CACb,6DAA6D,CAC9D,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,KAAc,EACd,QAA4B;IAE5B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,QAAQ;SACtB,KAAK,CAAC,CAAC,CAAC;SACR,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAChD,IACE,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QACpC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,EAC/D,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAAC,aAAqB;IAChD,MAAM,cAAc,GAAG,CAAC,CAAC,YAAY,CAAC,uBAAuB,EAAE;QAC7D,MAAM,EAAE,aAAa;KACtB,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CACpE,iBAAiB,CAAC,KAAK,CAAC,CACzB,CAAC;IACF,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,MAAM,CAAC,WAAW,CAC5B,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAC5C;gBACD,QAAQ,EAAE,KAAK;gBACf,oBAAoB,EAAE,KAAK;aAC5B;SACF;QACD,QAAQ,EAAE,CAAC,WAAW,CAAC;QACvB,oBAAoB,EAAE,KAAK;KAC5B,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,0BAA0B;IACR;IAA7B,YAA6B,MAAsC;QAAtC,WAAM,GAAN,MAAM,CAAgC;IAAG,CAAC;IAEvE,KAAK,CAAC,MAAM,CAAC,QAA4B;QACvC,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3B,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,cAAc;YACrB,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC;YACjC,MAAM,EAAE,mBAAmB,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YAChD,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,QAAQ,CAAC;SACxD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,QAA4B;QAC3C,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3B,KAAK,EAAE,aAAa;YACpB,KAAK,EAAE,aAAa;YACpB,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC;YAClC,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,GAAG,CAAC,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;aAC3D;YACD,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC;SACrD,CAAC,CAAC;IACL,CAAC;CACF","sourcesContent":["import { z } from \"incur\";\nimport { readFileSync } from \"node:fs\";\nimport Ajv2020, { type ValidateFunction } from \"ajv/dist/2020.js\";\nimport type { Finding } from \"../models.js\";\nimport type { CodexReviewRunner } from \"./codex-review.js\";\nimport { pairReviewPrompt, screeningPrompt } from \"./deduplication-prompts.js\";\n\nconst rationale = z.string().refine((value) => value.trim().length > 0);\nconst sameSchema = z.object({\n decision: z.literal(\"SAME\"),\n rationale,\n canonicalFindingId: z.string(),\n mergedFinding: z.record(z.string(), z.unknown()),\n});\nconst distinctSchema = z.object({\n decision: z.literal(\"DISTINCT\"),\n rationale,\n canonicalFindingId: z.null().optional(),\n mergedFinding: z.null().optional(),\n});\nconst reviewSchema = z.discriminatedUnion(\"decision\", [\n sameSchema,\n distinctSchema,\n]);\nconst screeningSameSchema = z.object({\n decision: z.literal(\"SAME\"),\n rationale,\n});\nconst screeningDistinctSchema = z.object({\n decision: z.literal(\"DISTINCT\"),\n rationale,\n});\nconst screeningDecisionSchema = z.discriminatedUnion(\"decision\", [\n screeningSameSchema.strict(),\n screeningDistinctSchema.strict(),\n]);\n// The host assigns exact slot names; finding IDs stay out of model output.\nconst screeningSchema = z\n .object({\n decisions: z.record(z.string(), screeningDecisionSchema),\n })\n .strict();\n\nlet validateMergedFinding: ValidateFunction<Finding> | undefined;\n\nfunction requireMergedFinding(result: DuplicateDecision): void {\n if (result.decision !== \"SAME\") return;\n if (validateMergedFinding === undefined) {\n const schema = JSON.parse(\n readFileSync(\n new URL(\n \"../../_bundled_plugin/schemas/findings.schema.json\",\n import.meta.url,\n ),\n \"utf8\",\n ),\n );\n validateMergedFinding = new Ajv2020({ strict: false }).compile<Finding>(\n schema.properties.findings.items,\n );\n }\n if (\n !validateMergedFinding(result.mergedFinding) ||\n result.mergedFinding[\"findingId\"] !== result.canonicalFindingId\n )\n throw new Error(\n \"Every SAME decision requires a generated mergedFinding in the Finding schema with the canonical finding's identity.\",\n );\n}\n\nexport type ScreeningResult = z.infer<typeof screeningSchema>;\nexport type DuplicateDecision = z.infer<typeof reviewSchema>;\n\nexport interface DeduplicationReviewer {\n screen(findings: readonly Finding[]): Promise<ScreeningResult>;\n reviewPair(findings: readonly Finding[]): Promise<DuplicateDecision>;\n}\n\nexport function pairKey(ids: readonly string[]): string {\n return JSON.stringify([...ids].sort());\n}\n\nexport function screeningPairSlot(index: number): string {\n return `pair-${index + 1}`;\n}\n\nexport function validateReview(\n value: unknown,\n findings: readonly Finding[],\n): DuplicateDecision {\n const result = reviewSchema.parse(value);\n requireMergedFinding(result);\n if (\n result.decision === \"SAME\" &&\n !findings.some((finding) => finding.findingId === result.canonicalFindingId)\n ) {\n throw new Error(\n \"The canonical finding must belong to the assigned findings.\",\n );\n }\n return result;\n}\n\nexport function validateScreening(\n value: unknown,\n findings: readonly Finding[],\n): ScreeningResult {\n const result = screeningSchema.parse(value);\n const required = findings\n .slice(1)\n .map((_finding, index) => screeningPairSlot(index));\n const submitted = Object.keys(result.decisions);\n if (\n submitted.length !== required.length ||\n required.some((slot) => !Object.hasOwn(result.decisions, slot))\n ) {\n throw new Error(\"Submit exactly the assigned screening pair slots.\");\n }\n return result;\n}\n\nfunction screeningToolSchema(neighborCount: number): object {\n const decisionSchema = z.toJSONSchema(screeningDecisionSchema, {\n target: \"openapi-3.0\",\n });\n const slots = Array.from({ length: neighborCount }, (_value, index) =>\n screeningPairSlot(index),\n );\n return {\n type: \"object\",\n properties: {\n decisions: {\n type: \"object\",\n properties: Object.fromEntries(\n slots.map((slot) => [slot, decisionSchema]),\n ),\n required: slots,\n additionalProperties: false,\n },\n },\n required: [\"decisions\"],\n additionalProperties: false,\n };\n}\n\nexport class CodexDeduplicationReviewer implements DeduplicationReviewer {\n constructor(private readonly runner: Pick<CodexReviewRunner, \"run\">) {}\n\n async screen(findings: readonly Finding[]): Promise<ScreeningResult> {\n return await this.runner.run({\n stage: \"screening\",\n model: \"gpt-5.6-luna\",\n effort: \"xhigh\",\n prompt: screeningPrompt(findings),\n schema: screeningToolSchema(findings.length - 1),\n validate: (value) => validateScreening(value, findings),\n });\n }\n\n async reviewPair(findings: readonly Finding[]): Promise<DuplicateDecision> {\n return await this.runner.run({\n stage: \"pair-review\",\n model: \"gpt-5.6-sol\",\n effort: \"xhigh\",\n prompt: pairReviewPrompt(findings),\n schema: {\n type: \"object\",\n ...z.toJSONSchema(reviewSchema, { target: \"openapi-3.0\" }),\n },\n validate: (value) => validateReview(value, findings),\n });\n }\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deduplication.d.ts","sourceRoot":"","sources":["../../src/deduplication/deduplication.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"deduplication.d.ts","sourceRoot":"","sources":["../../src/deduplication/deduplication.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,mBAAmB;IAClC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,eAAe,EAAE,MAAM,EAAE,EAAE,CAAC;IAC5B,mBAAmB,EAAE,WAAW,CAAC;CAClC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { pairKey, } from "./deduplication-reviewer.js";
|
|
1
|
+
import { pairKey, screeningPairSlot, } from "./deduplication-reviewer.js";
|
|
2
2
|
const severityOrder = {
|
|
3
3
|
critical: 0,
|
|
4
4
|
high: 1,
|
|
@@ -6,6 +6,126 @@ const severityOrder = {
|
|
|
6
6
|
low: 3,
|
|
7
7
|
informational: 4,
|
|
8
8
|
};
|
|
9
|
+
function scoreAfter(left, right) {
|
|
10
|
+
for (let index = 0; index < left.length; index++) {
|
|
11
|
+
if (left[index] !== right[index])
|
|
12
|
+
return left[index] > right[index];
|
|
13
|
+
}
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
function addSupport(support, left, right) {
|
|
17
|
+
const leftSupport = support.get(left) ?? new Map();
|
|
18
|
+
const rightSupport = support.get(right) ?? new Map();
|
|
19
|
+
leftSupport.set(right, (leftSupport.get(right) ?? 0) + 1);
|
|
20
|
+
rightSupport.set(left, (rightSupport.get(left) ?? 0) + 1);
|
|
21
|
+
support.set(left, leftSupport);
|
|
22
|
+
support.set(right, rightSupport);
|
|
23
|
+
}
|
|
24
|
+
function addConflict(conflicts, left, right) {
|
|
25
|
+
const leftConflicts = conflicts.get(left) ?? new Set();
|
|
26
|
+
const rightConflicts = conflicts.get(right) ?? new Set();
|
|
27
|
+
leftConflicts.add(right);
|
|
28
|
+
rightConflicts.add(left);
|
|
29
|
+
conflicts.set(left, leftConflicts);
|
|
30
|
+
conflicts.set(right, rightConflicts);
|
|
31
|
+
}
|
|
32
|
+
/** Greedily retain the best-supported legal merges, using input order for ties. */
|
|
33
|
+
/** @internal */
|
|
34
|
+
export function contradictionFreeSubgroups(findingIds, samePairs, distinctPairs, signal, metrics) {
|
|
35
|
+
const indexes = new Map(findingIds.map((findingId, index) => [findingId, index]));
|
|
36
|
+
const active = new Set(findingIds.map((_findingId, index) => index));
|
|
37
|
+
const clusters = new Map(findingIds.map((findingId, index) => [index, new Set([findingId])]));
|
|
38
|
+
const support = new Map();
|
|
39
|
+
for (const [left, right] of samePairs)
|
|
40
|
+
addSupport(support, indexes.get(left), indexes.get(right));
|
|
41
|
+
const conflicts = new Map();
|
|
42
|
+
for (const [left, right] of distinctPairs)
|
|
43
|
+
addConflict(conflicts, indexes.get(left), indexes.get(right));
|
|
44
|
+
while (true) {
|
|
45
|
+
signal?.throwIfAborted();
|
|
46
|
+
let selected;
|
|
47
|
+
let selectedScore;
|
|
48
|
+
for (const leftCluster of active) {
|
|
49
|
+
for (const [rightCluster, gain] of support.get(leftCluster) ?? []) {
|
|
50
|
+
if (leftCluster >= rightCluster ||
|
|
51
|
+
!active.has(rightCluster) ||
|
|
52
|
+
conflicts.get(leftCluster)?.has(rightCluster))
|
|
53
|
+
continue;
|
|
54
|
+
if (metrics)
|
|
55
|
+
metrics.candidateEvaluations++;
|
|
56
|
+
let newlyBlocked = 0;
|
|
57
|
+
const leftConflicts = conflicts.get(leftCluster);
|
|
58
|
+
const rightConflicts = conflicts.get(rightCluster);
|
|
59
|
+
for (const other of leftConflicts ?? []) {
|
|
60
|
+
if (metrics)
|
|
61
|
+
metrics.conflictNeighborChecks++;
|
|
62
|
+
if (other !== rightCluster && !rightConflicts?.has(other))
|
|
63
|
+
newlyBlocked += support.get(rightCluster)?.get(other) ?? 0;
|
|
64
|
+
}
|
|
65
|
+
for (const other of rightConflicts ?? []) {
|
|
66
|
+
if (metrics)
|
|
67
|
+
metrics.conflictNeighborChecks++;
|
|
68
|
+
if (other !== leftCluster && !leftConflicts?.has(other))
|
|
69
|
+
newlyBlocked += support.get(leftCluster)?.get(other) ?? 0;
|
|
70
|
+
}
|
|
71
|
+
const score = [
|
|
72
|
+
gain - newlyBlocked,
|
|
73
|
+
gain,
|
|
74
|
+
-newlyBlocked,
|
|
75
|
+
-leftCluster,
|
|
76
|
+
-rightCluster,
|
|
77
|
+
];
|
|
78
|
+
if (selectedScore === undefined || scoreAfter(score, selectedScore)) {
|
|
79
|
+
selected = [leftCluster, rightCluster];
|
|
80
|
+
selectedScore = score;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (selected === undefined)
|
|
85
|
+
return [...active]
|
|
86
|
+
.map((cluster) => clusters.get(cluster))
|
|
87
|
+
.filter((members) => members.size > 1);
|
|
88
|
+
const [leftCluster, rightCluster] = selected;
|
|
89
|
+
for (const member of clusters.get(rightCluster))
|
|
90
|
+
clusters.get(leftCluster).add(member);
|
|
91
|
+
clusters.delete(rightCluster);
|
|
92
|
+
active.delete(rightCluster);
|
|
93
|
+
const leftSupport = support.get(leftCluster) ?? new Map();
|
|
94
|
+
const rightSupport = support.get(rightCluster) ?? new Map();
|
|
95
|
+
const supportNeighbors = new Set([
|
|
96
|
+
...leftSupport.keys(),
|
|
97
|
+
...rightSupport.keys(),
|
|
98
|
+
]);
|
|
99
|
+
supportNeighbors.delete(leftCluster);
|
|
100
|
+
supportNeighbors.delete(rightCluster);
|
|
101
|
+
leftSupport.delete(rightCluster);
|
|
102
|
+
for (const neighbor of supportNeighbors) {
|
|
103
|
+
const weight = (leftSupport.get(neighbor) ?? 0) + (rightSupport.get(neighbor) ?? 0);
|
|
104
|
+
leftSupport.set(neighbor, weight);
|
|
105
|
+
const neighborSupport = support.get(neighbor);
|
|
106
|
+
neighborSupport.delete(rightCluster);
|
|
107
|
+
neighborSupport.set(leftCluster, weight);
|
|
108
|
+
}
|
|
109
|
+
support.set(leftCluster, leftSupport);
|
|
110
|
+
support.delete(rightCluster);
|
|
111
|
+
const mergedConflicts = new Set([
|
|
112
|
+
...(conflicts.get(leftCluster) ?? []),
|
|
113
|
+
...(conflicts.get(rightCluster) ?? []),
|
|
114
|
+
]);
|
|
115
|
+
mergedConflicts.delete(leftCluster);
|
|
116
|
+
mergedConflicts.delete(rightCluster);
|
|
117
|
+
for (const neighbor of mergedConflicts) {
|
|
118
|
+
const neighborConflicts = conflicts.get(neighbor);
|
|
119
|
+
neighborConflicts.delete(rightCluster);
|
|
120
|
+
neighborConflicts.add(leftCluster);
|
|
121
|
+
}
|
|
122
|
+
if (mergedConflicts.size > 0)
|
|
123
|
+
conflicts.set(leftCluster, mergedConflicts);
|
|
124
|
+
else
|
|
125
|
+
conflicts.delete(leftCluster);
|
|
126
|
+
conflicts.delete(rightCluster);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
9
129
|
/** @internal */
|
|
10
130
|
export class FindingDeduplicator {
|
|
11
131
|
candidates;
|
|
@@ -21,6 +141,7 @@ export class FindingDeduplicator {
|
|
|
21
141
|
const ids = [...new Set(findingIds)];
|
|
22
142
|
const findings = new Map();
|
|
23
143
|
const nominated = new Map();
|
|
144
|
+
const rejected = new Map();
|
|
24
145
|
for (const id of ids) {
|
|
25
146
|
this.signal?.throwIfAborted();
|
|
26
147
|
const result = await this.candidates.potentialDuplicates(id);
|
|
@@ -30,50 +151,92 @@ export class FindingDeduplicator {
|
|
|
30
151
|
if (neighborhood.length < 2)
|
|
31
152
|
continue;
|
|
32
153
|
const screening = await this.reviewer.screen(neighborhood);
|
|
33
|
-
for (
|
|
154
|
+
for (let index = 0; index < neighborhood.length - 1; index++) {
|
|
155
|
+
const decision = screening.decisions[screeningPairSlot(index)];
|
|
156
|
+
const pair = [
|
|
157
|
+
neighborhood[0].findingId,
|
|
158
|
+
neighborhood[index + 1].findingId,
|
|
159
|
+
];
|
|
160
|
+
const key = pairKey(pair);
|
|
34
161
|
if (decision.decision === "SAME") {
|
|
35
|
-
|
|
162
|
+
if (!rejected.has(key))
|
|
163
|
+
nominated.set(key, pair);
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
rejected.set(key, pair);
|
|
167
|
+
nominated.delete(key);
|
|
36
168
|
}
|
|
37
169
|
}
|
|
38
170
|
}
|
|
39
|
-
const
|
|
171
|
+
const supported = [];
|
|
40
172
|
for (const pair of nominated.values()) {
|
|
41
173
|
this.signal?.throwIfAborted();
|
|
42
174
|
const originals = pair.map((id) => findings.get(id));
|
|
43
|
-
if ((await this.reviewer.reviewPair(originals)).decision
|
|
44
|
-
|
|
175
|
+
if ((await this.reviewer.reviewPair(originals)).decision === "SAME") {
|
|
176
|
+
supported.push(pair);
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
rejected.set(pairKey(pair), pair);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
const adjacent = new Map();
|
|
183
|
+
for (const pair of supported) {
|
|
45
184
|
for (const [left, right] of [pair, [pair[1], pair[0]]]) {
|
|
46
185
|
const neighbors = adjacent.get(left) ?? new Set();
|
|
47
186
|
neighbors.add(right);
|
|
48
187
|
adjacent.set(left, neighbors);
|
|
49
188
|
}
|
|
50
189
|
}
|
|
51
|
-
const
|
|
52
|
-
const
|
|
53
|
-
const duplicateGroups = [];
|
|
54
|
-
const canonical = new Map();
|
|
190
|
+
const components = [];
|
|
191
|
+
const componentByFinding = new Map();
|
|
55
192
|
for (const id of findings.keys()) {
|
|
56
193
|
this.signal?.throwIfAborted();
|
|
57
|
-
if (!adjacent.has(id) ||
|
|
194
|
+
if (!adjacent.has(id) || componentByFinding.has(id))
|
|
58
195
|
continue;
|
|
59
|
-
const
|
|
196
|
+
const component = {
|
|
197
|
+
members: [],
|
|
198
|
+
supported: [],
|
|
199
|
+
rejected: [],
|
|
200
|
+
};
|
|
201
|
+
components.push(component);
|
|
60
202
|
const pending = [id];
|
|
61
203
|
while (pending.length > 0) {
|
|
62
204
|
const member = pending.pop();
|
|
63
|
-
if (
|
|
205
|
+
if (componentByFinding.has(member))
|
|
64
206
|
continue;
|
|
65
|
-
|
|
66
|
-
members.push(member);
|
|
207
|
+
componentByFinding.set(member, component);
|
|
67
208
|
pending.push(...adjacent.get(member));
|
|
68
209
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
210
|
+
}
|
|
211
|
+
// Preserve finding insertion order for contradiction-grouping ties.
|
|
212
|
+
for (const id of findings.keys())
|
|
213
|
+
componentByFinding.get(id)?.members.push(id);
|
|
214
|
+
for (const pair of supported)
|
|
215
|
+
componentByFinding.get(pair[0]).supported.push(pair);
|
|
216
|
+
for (const pair of rejected.values()) {
|
|
217
|
+
const component = componentByFinding.get(pair[0]);
|
|
218
|
+
if (component && component === componentByFinding.get(pair[1]))
|
|
219
|
+
component.rejected.push(pair);
|
|
220
|
+
}
|
|
221
|
+
const selected = new Set(ids);
|
|
222
|
+
const duplicateGroups = [];
|
|
223
|
+
const canonical = new Map();
|
|
224
|
+
for (const component of components) {
|
|
225
|
+
this.signal?.throwIfAborted();
|
|
226
|
+
const groups = component.rejected.length === 0
|
|
227
|
+
? [new Set(component.members)]
|
|
228
|
+
: contradictionFreeSubgroups(component.members, component.supported, component.rejected, this.signal);
|
|
229
|
+
for (const group of groups) {
|
|
230
|
+
const groupMembers = [...group];
|
|
231
|
+
if (!groupMembers.some((member) => selected.has(member)))
|
|
232
|
+
continue;
|
|
233
|
+
groupMembers.sort((left, right) => severityOrder[findings.get(left).severity.level] -
|
|
234
|
+
severityOrder[findings.get(right).severity.level] ||
|
|
235
|
+
(left < right ? -1 : left > right ? 1 : 0));
|
|
236
|
+
duplicateGroups.push(groupMembers);
|
|
237
|
+
for (const member of groupMembers)
|
|
238
|
+
canonical.set(member, groupMembers[0]);
|
|
239
|
+
}
|
|
77
240
|
}
|
|
78
241
|
return {
|
|
79
242
|
uniqueFindingIds: [...new Set(ids.map((id) => canonical.get(id) ?? id))],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deduplication.js","sourceRoot":"","sources":["../../src/deduplication/deduplication.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,OAAO,GAER,MAAM,6BAA6B,CAAC;AAQrC,MAAM,aAAa,GAAiD;IAClE,QAAQ,EAAE,CAAC;IACX,IAAI,EAAE,CAAC;IACP,MAAM,EAAE,CAAC;IACT,GAAG,EAAE,CAAC;IACN,aAAa,EAAE,CAAC;CACjB,CAAC;AAEF,gBAAgB;AAChB,MAAM,OAAO,mBAAmB;IAEX;IAGA;IACA;IALnB,YACmB,UAEhB,EACgB,QAA+B,EAC/B,MAAoB;QAJpB,eAAU,GAAV,UAAU,CAE1B;QACgB,aAAQ,GAAR,QAAQ,CAAuB;QAC/B,WAAM,GAAN,MAAM,CAAc;IACpC,CAAC;IAEJ,KAAK,CAAC,GAAG,CAAC,UAA6B;QACrC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4B,CAAC;QACtD,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YAC7D,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACrE,KAAK,MAAM,OAAO,IAAI,YAAY;gBAChC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAC3C,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YACtC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3D,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC3C,IAAI,QAAQ,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;oBACjC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;gBACnE,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC;YACtD,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM;gBACjE,SAAS;YACX,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAU,EAAE,CAAC;gBAChE,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;gBAC1D,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACrB,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,eAAe,GAAe,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YACnD,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC;YACrB,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,EAAG,CAAC;gBAC9B,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;oBAAE,SAAS;gBAClC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAAE,SAAS;YAC9D,OAAO,CAAC,IAAI,CACV,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAC/C,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACpD,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7C,CAAC;YACF,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,KAAK,MAAM,MAAM,IAAI,OAAO;gBAAE,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;QACnE,CAAC;QACD,OAAO;YACL,gBAAgB,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACxE,eAAe;YACf,mBAAmB,EAAE,WAAW;SACjC,CAAC;IACJ,CAAC;CACF","sourcesContent":["import type { Finding } from \"../models.js\";\nimport type { FindingNeighborhood } from \"../finding-retrieval.js\";\nimport {\n pairKey,\n type DeduplicationReviewer,\n} from \"./deduplication-reviewer.js\";\n\nexport interface DeduplicationResult {\n uniqueFindingIds: string[];\n duplicateGroups: string[][];\n deduplicationStatus: \"completed\";\n}\n\nconst severityOrder: Record<Finding[\"severity\"][\"level\"], number> = {\n critical: 0,\n high: 1,\n medium: 2,\n low: 3,\n informational: 4,\n};\n\n/** @internal */\nexport class FindingDeduplicator {\n constructor(\n private readonly candidates: {\n potentialDuplicates(findingId: string): Promise<FindingNeighborhood>;\n },\n private readonly reviewer: DeduplicationReviewer,\n private readonly signal?: AbortSignal,\n ) {}\n\n async run(findingIds: readonly string[]): Promise<DeduplicationResult> {\n this.signal?.throwIfAborted();\n const ids = [...new Set(findingIds)];\n const findings = new Map<string, Finding>();\n const nominated = new Map<string, [string, string]>();\n for (const id of ids) {\n this.signal?.throwIfAborted();\n const result = await this.candidates.potentialDuplicates(id);\n const neighborhood = [result.finding, ...result.potentialDuplicates];\n for (const finding of neighborhood)\n findings.set(finding.findingId, finding);\n if (neighborhood.length < 2) continue;\n const screening = await this.reviewer.screen(neighborhood);\n for (const decision of screening.decisions) {\n if (decision.decision === \"SAME\") {\n nominated.set(pairKey(decision.findingIds), decision.findingIds);\n }\n }\n }\n\n const adjacent = new Map<string, Set<string>>();\n for (const pair of nominated.values()) {\n this.signal?.throwIfAborted();\n const originals = pair.map((id) => findings.get(id)!);\n if ((await this.reviewer.reviewPair(originals)).decision !== \"SAME\")\n continue;\n for (const [left, right] of [pair, [pair[1], pair[0]]] as const) {\n const neighbors = adjacent.get(left) ?? new Set<string>();\n neighbors.add(right);\n adjacent.set(left, neighbors);\n }\n }\n\n const selected = new Set(ids);\n const visited = new Set<string>();\n const duplicateGroups: string[][] = [];\n const canonical = new Map<string, string>();\n for (const id of findings.keys()) {\n this.signal?.throwIfAborted();\n if (!adjacent.has(id) || visited.has(id)) continue;\n const members: string[] = [];\n const pending = [id];\n while (pending.length > 0) {\n const member = pending.pop()!;\n if (visited.has(member)) continue;\n visited.add(member);\n members.push(member);\n pending.push(...adjacent.get(member)!);\n }\n if (!members.some((member) => selected.has(member))) continue;\n members.sort(\n (left, right) =>\n severityOrder[findings.get(left)!.severity.level] -\n severityOrder[findings.get(right)!.severity.level] ||\n (left < right ? -1 : left > right ? 1 : 0),\n );\n duplicateGroups.push(members);\n for (const member of members) canonical.set(member, members[0]!);\n }\n return {\n uniqueFindingIds: [...new Set(ids.map((id) => canonical.get(id) ?? id))],\n duplicateGroups,\n deduplicationStatus: \"completed\",\n };\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"deduplication.js","sourceRoot":"","sources":["../../src/deduplication/deduplication.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,OAAO,EACP,iBAAiB,GAElB,MAAM,6BAA6B,CAAC;AAQrC,MAAM,aAAa,GAAiD;IAClE,QAAQ,EAAE,CAAC;IACX,IAAI,EAAE,CAAC;IACP,MAAM,EAAE,CAAC;IACT,GAAG,EAAE,CAAC;IACN,aAAa,EAAE,CAAC;CACjB,CAAC;AAEF,SAAS,UAAU,CACjB,IAAuB,EACvB,KAAwB;IAExB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACjD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC,KAAK,CAAE,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC;IACxE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAQD,SAAS,UAAU,CACjB,OAAyC,EACzC,IAAY,EACZ,KAAa;IAEb,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;IACnE,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;IACrE,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,WAAW,CAClB,SAAmC,EACnC,IAAY,EACZ,KAAa;IAEb,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;IAC/D,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;IACjE,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACzB,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IACnC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;AACvC,CAAC;AAED,mFAAmF;AACnF,gBAAgB;AAChB,MAAM,UAAU,0BAA0B,CACxC,UAA6B,EAC7B,SAAiD,EACjD,aAAqD,EACrD,MAAoB,EACpB,OAAsC;IAEtC,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CACzD,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CACpE,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,GAAG,EAA+B,CAAC;IACvD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,SAAS;QACnC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAE,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;IACjD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,aAAa;QACvC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAE,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,CAAC;IAElE,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,IAAI,QAA+C,CAAC;QACpD,IAAI,aAA4C,CAAC;QACjD,KAAK,MAAM,WAAW,IAAI,MAAM,EAAE,CAAC;YACjC,KAAK,MAAM,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;gBAClE,IACE,WAAW,IAAI,YAAY;oBAC3B,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC;oBACzB,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,YAAY,CAAC;oBAE7C,SAAS;gBACX,IAAI,OAAO;oBAAE,OAAO,CAAC,oBAAoB,EAAE,CAAC;gBAC5C,IAAI,YAAY,GAAG,CAAC,CAAC;gBACrB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;gBACjD,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;gBACnD,KAAK,MAAM,KAAK,IAAI,aAAa,IAAI,EAAE,EAAE,CAAC;oBACxC,IAAI,OAAO;wBAAE,OAAO,CAAC,sBAAsB,EAAE,CAAC;oBAC9C,IAAI,KAAK,KAAK,YAAY,IAAI,CAAC,cAAc,EAAE,GAAG,CAAC,KAAK,CAAC;wBACvD,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC/D,CAAC;gBACD,KAAK,MAAM,KAAK,IAAI,cAAc,IAAI,EAAE,EAAE,CAAC;oBACzC,IAAI,OAAO;wBAAE,OAAO,CAAC,sBAAsB,EAAE,CAAC;oBAC9C,IAAI,KAAK,KAAK,WAAW,IAAI,CAAC,aAAa,EAAE,GAAG,CAAC,KAAK,CAAC;wBACrD,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC9D,CAAC;gBACD,MAAM,KAAK,GAAG;oBACZ,IAAI,GAAG,YAAY;oBACnB,IAAI;oBACJ,CAAC,YAAY;oBACb,CAAC,WAAW;oBACZ,CAAC,YAAY;iBACd,CAAC;gBACF,IAAI,aAAa,KAAK,SAAS,IAAI,UAAU,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,CAAC;oBACpE,QAAQ,GAAG,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;oBACvC,aAAa,GAAG,KAAK,CAAC;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,QAAQ,KAAK,SAAS;YACxB,OAAO,CAAC,GAAG,MAAM,CAAC;iBACf,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;iBACxC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC,GAAG,QAAQ,CAAC;QAC7C,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAE;YAC9C,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAE5B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;QAC1E,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;QAC5E,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;YAC/B,GAAG,WAAW,CAAC,IAAI,EAAE;YACrB,GAAG,YAAY,CAAC,IAAI,EAAE;SACvB,CAAC,CAAC;QACH,gBAAgB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACrC,gBAAgB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACtC,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACjC,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;YACxC,MAAM,MAAM,GACV,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YACvE,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAClC,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YAC/C,eAAe,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACrC,eAAe,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACtC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAE7B,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;YAC9B,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YACrC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;SACvC,CAAC,CAAC;QACH,eAAe,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACpC,eAAe,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACrC,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,iBAAiB,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YACnD,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACvC,iBAAiB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,eAAe,CAAC,IAAI,GAAG,CAAC;YAAE,SAAS,CAAC,GAAG,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;;YACrE,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACnC,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,gBAAgB;AAChB,MAAM,OAAO,mBAAmB;IAEX;IAGA;IACA;IALnB,YACmB,UAEhB,EACgB,QAA+B,EAC/B,MAAoB;QAJpB,eAAU,GAAV,UAAU,CAE1B;QACgB,aAAQ,GAAR,QAAQ,CAAuB;QAC/B,WAAM,GAAN,MAAM,CAAc;IACpC,CAAC;IAEJ,KAAK,CAAC,GAAG,CAAC,UAA6B;QACrC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4B,CAAC;QACtD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4B,CAAC;QACrD,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YAC7D,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACrE,KAAK,MAAM,OAAO,IAAI,YAAY;gBAChC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAC3C,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YACtC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3D,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC7D,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAE,CAAC;gBAChE,MAAM,IAAI,GAAqB;oBAC7B,YAAY,CAAC,CAAC,CAAE,CAAC,SAAS;oBAC1B,YAAY,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,SAAS;iBACnC,CAAC;gBACF,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;gBAC1B,IAAI,QAAQ,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;oBACjC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBACnD,CAAC;qBAAM,CAAC;oBACN,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;oBACxB,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAuB,EAAE,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC;YACtD,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;gBACpE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAU,EAAE,CAAC;gBAChE,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;gBAC1D,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACrB,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAIV,EAAE,CAAC;QACT,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAuC,CAAC;QAC1E,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YAC9D,MAAM,SAAS,GAAgC;gBAC7C,OAAO,EAAE,EAAE;gBACX,SAAS,EAAE,EAAE;gBACb,QAAQ,EAAE,EAAE;aACb,CAAC;YACF,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC3B,MAAM,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC;YACrB,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,EAAG,CAAC;gBAC9B,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC;oBAAE,SAAS;gBAC7C,kBAAkB,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QACD,oEAAoE;QACpE,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,EAAE;YAC9B,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/C,KAAK,MAAM,IAAI,IAAI,SAAS;YAC1B,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACrC,MAAM,SAAS,GAAG,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,IAAI,SAAS,IAAI,SAAS,KAAK,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC5D,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,eAAe,GAAe,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,MAAM,MAAM,GACV,SAAS,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAC7B,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBAC9B,CAAC,CAAC,0BAA0B,CACxB,SAAS,CAAC,OAAO,EACjB,SAAS,CAAC,SAAS,EACnB,SAAS,CAAC,QAAQ,EAClB,IAAI,CAAC,MAAM,CACZ,CAAC;YACR,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,YAAY,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;gBAChC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAAE,SAAS;gBACnE,YAAY,CAAC,IAAI,CACf,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;oBAC/C,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;oBACpD,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7C,CAAC;gBACF,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBACnC,KAAK,MAAM,MAAM,IAAI,YAAY;oBAC/B,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO;YACL,gBAAgB,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACxE,eAAe;YACf,mBAAmB,EAAE,WAAW;SACjC,CAAC;IACJ,CAAC;CACF","sourcesContent":["import type { Finding } from \"../models.js\";\nimport type { FindingNeighborhood } from \"../finding-retrieval.js\";\nimport {\n pairKey,\n screeningPairSlot,\n type DeduplicationReviewer,\n} from \"./deduplication-reviewer.js\";\n\nexport interface DeduplicationResult {\n uniqueFindingIds: string[];\n duplicateGroups: string[][];\n deduplicationStatus: \"completed\";\n}\n\nconst severityOrder: Record<Finding[\"severity\"][\"level\"], number> = {\n critical: 0,\n high: 1,\n medium: 2,\n low: 3,\n informational: 4,\n};\n\nfunction scoreAfter(\n left: readonly number[],\n right: readonly number[],\n): boolean {\n for (let index = 0; index < left.length; index++) {\n if (left[index] !== right[index]) return left[index]! > right[index]!;\n }\n return false;\n}\n\n/** @internal */\nexport interface ContradictionGroupingMetrics {\n candidateEvaluations: number;\n conflictNeighborChecks: number;\n}\n\nfunction addSupport(\n support: Map<number, Map<number, number>>,\n left: number,\n right: number,\n): void {\n const leftSupport = support.get(left) ?? new Map<number, number>();\n const rightSupport = support.get(right) ?? new Map<number, number>();\n leftSupport.set(right, (leftSupport.get(right) ?? 0) + 1);\n rightSupport.set(left, (rightSupport.get(left) ?? 0) + 1);\n support.set(left, leftSupport);\n support.set(right, rightSupport);\n}\n\nfunction addConflict(\n conflicts: Map<number, Set<number>>,\n left: number,\n right: number,\n): void {\n const leftConflicts = conflicts.get(left) ?? new Set<number>();\n const rightConflicts = conflicts.get(right) ?? new Set<number>();\n leftConflicts.add(right);\n rightConflicts.add(left);\n conflicts.set(left, leftConflicts);\n conflicts.set(right, rightConflicts);\n}\n\n/** Greedily retain the best-supported legal merges, using input order for ties. */\n/** @internal */\nexport function contradictionFreeSubgroups(\n findingIds: readonly string[],\n samePairs: readonly (readonly [string, string])[],\n distinctPairs: readonly (readonly [string, string])[],\n signal?: AbortSignal,\n metrics?: ContradictionGroupingMetrics,\n): Set<string>[] {\n const indexes = new Map(\n findingIds.map((findingId, index) => [findingId, index]),\n );\n const active = new Set(findingIds.map((_findingId, index) => index));\n const clusters = new Map(\n findingIds.map((findingId, index) => [index, new Set([findingId])]),\n );\n const support = new Map<number, Map<number, number>>();\n for (const [left, right] of samePairs)\n addSupport(support, indexes.get(left)!, indexes.get(right)!);\n const conflicts = new Map<number, Set<number>>();\n for (const [left, right] of distinctPairs)\n addConflict(conflicts, indexes.get(left)!, indexes.get(right)!);\n\n while (true) {\n signal?.throwIfAborted();\n let selected: readonly [number, number] | undefined;\n let selectedScore: readonly number[] | undefined;\n for (const leftCluster of active) {\n for (const [rightCluster, gain] of support.get(leftCluster) ?? []) {\n if (\n leftCluster >= rightCluster ||\n !active.has(rightCluster) ||\n conflicts.get(leftCluster)?.has(rightCluster)\n )\n continue;\n if (metrics) metrics.candidateEvaluations++;\n let newlyBlocked = 0;\n const leftConflicts = conflicts.get(leftCluster);\n const rightConflicts = conflicts.get(rightCluster);\n for (const other of leftConflicts ?? []) {\n if (metrics) metrics.conflictNeighborChecks++;\n if (other !== rightCluster && !rightConflicts?.has(other))\n newlyBlocked += support.get(rightCluster)?.get(other) ?? 0;\n }\n for (const other of rightConflicts ?? []) {\n if (metrics) metrics.conflictNeighborChecks++;\n if (other !== leftCluster && !leftConflicts?.has(other))\n newlyBlocked += support.get(leftCluster)?.get(other) ?? 0;\n }\n const score = [\n gain - newlyBlocked,\n gain,\n -newlyBlocked,\n -leftCluster,\n -rightCluster,\n ];\n if (selectedScore === undefined || scoreAfter(score, selectedScore)) {\n selected = [leftCluster, rightCluster];\n selectedScore = score;\n }\n }\n }\n if (selected === undefined)\n return [...active]\n .map((cluster) => clusters.get(cluster)!)\n .filter((members) => members.size > 1);\n const [leftCluster, rightCluster] = selected;\n for (const member of clusters.get(rightCluster)!)\n clusters.get(leftCluster)!.add(member);\n clusters.delete(rightCluster);\n active.delete(rightCluster);\n\n const leftSupport = support.get(leftCluster) ?? new Map<number, number>();\n const rightSupport = support.get(rightCluster) ?? new Map<number, number>();\n const supportNeighbors = new Set([\n ...leftSupport.keys(),\n ...rightSupport.keys(),\n ]);\n supportNeighbors.delete(leftCluster);\n supportNeighbors.delete(rightCluster);\n leftSupport.delete(rightCluster);\n for (const neighbor of supportNeighbors) {\n const weight =\n (leftSupport.get(neighbor) ?? 0) + (rightSupport.get(neighbor) ?? 0);\n leftSupport.set(neighbor, weight);\n const neighborSupport = support.get(neighbor)!;\n neighborSupport.delete(rightCluster);\n neighborSupport.set(leftCluster, weight);\n }\n support.set(leftCluster, leftSupport);\n support.delete(rightCluster);\n\n const mergedConflicts = new Set([\n ...(conflicts.get(leftCluster) ?? []),\n ...(conflicts.get(rightCluster) ?? []),\n ]);\n mergedConflicts.delete(leftCluster);\n mergedConflicts.delete(rightCluster);\n for (const neighbor of mergedConflicts) {\n const neighborConflicts = conflicts.get(neighbor)!;\n neighborConflicts.delete(rightCluster);\n neighborConflicts.add(leftCluster);\n }\n if (mergedConflicts.size > 0) conflicts.set(leftCluster, mergedConflicts);\n else conflicts.delete(leftCluster);\n conflicts.delete(rightCluster);\n }\n}\n\n/** @internal */\nexport class FindingDeduplicator {\n constructor(\n private readonly candidates: {\n potentialDuplicates(findingId: string): Promise<FindingNeighborhood>;\n },\n private readonly reviewer: DeduplicationReviewer,\n private readonly signal?: AbortSignal,\n ) {}\n\n async run(findingIds: readonly string[]): Promise<DeduplicationResult> {\n this.signal?.throwIfAborted();\n const ids = [...new Set(findingIds)];\n const findings = new Map<string, Finding>();\n const nominated = new Map<string, [string, string]>();\n const rejected = new Map<string, [string, string]>();\n for (const id of ids) {\n this.signal?.throwIfAborted();\n const result = await this.candidates.potentialDuplicates(id);\n const neighborhood = [result.finding, ...result.potentialDuplicates];\n for (const finding of neighborhood)\n findings.set(finding.findingId, finding);\n if (neighborhood.length < 2) continue;\n const screening = await this.reviewer.screen(neighborhood);\n for (let index = 0; index < neighborhood.length - 1; index++) {\n const decision = screening.decisions[screeningPairSlot(index)]!;\n const pair: [string, string] = [\n neighborhood[0]!.findingId,\n neighborhood[index + 1]!.findingId,\n ];\n const key = pairKey(pair);\n if (decision.decision === \"SAME\") {\n if (!rejected.has(key)) nominated.set(key, pair);\n } else {\n rejected.set(key, pair);\n nominated.delete(key);\n }\n }\n }\n\n const supported: [string, string][] = [];\n for (const pair of nominated.values()) {\n this.signal?.throwIfAborted();\n const originals = pair.map((id) => findings.get(id)!);\n if ((await this.reviewer.reviewPair(originals)).decision === \"SAME\") {\n supported.push(pair);\n } else {\n rejected.set(pairKey(pair), pair);\n }\n }\n\n const adjacent = new Map<string, Set<string>>();\n for (const pair of supported) {\n for (const [left, right] of [pair, [pair[1], pair[0]]] as const) {\n const neighbors = adjacent.get(left) ?? new Set<string>();\n neighbors.add(right);\n adjacent.set(left, neighbors);\n }\n }\n\n const components: {\n members: string[];\n supported: [string, string][];\n rejected: [string, string][];\n }[] = [];\n const componentByFinding = new Map<string, (typeof components)[number]>();\n for (const id of findings.keys()) {\n this.signal?.throwIfAborted();\n if (!adjacent.has(id) || componentByFinding.has(id)) continue;\n const component: (typeof components)[number] = {\n members: [],\n supported: [],\n rejected: [],\n };\n components.push(component);\n const pending = [id];\n while (pending.length > 0) {\n const member = pending.pop()!;\n if (componentByFinding.has(member)) continue;\n componentByFinding.set(member, component);\n pending.push(...adjacent.get(member)!);\n }\n }\n // Preserve finding insertion order for contradiction-grouping ties.\n for (const id of findings.keys())\n componentByFinding.get(id)?.members.push(id);\n for (const pair of supported)\n componentByFinding.get(pair[0])!.supported.push(pair);\n for (const pair of rejected.values()) {\n const component = componentByFinding.get(pair[0]);\n if (component && component === componentByFinding.get(pair[1]))\n component.rejected.push(pair);\n }\n\n const selected = new Set(ids);\n const duplicateGroups: string[][] = [];\n const canonical = new Map<string, string>();\n for (const component of components) {\n this.signal?.throwIfAborted();\n const groups =\n component.rejected.length === 0\n ? [new Set(component.members)]\n : contradictionFreeSubgroups(\n component.members,\n component.supported,\n component.rejected,\n this.signal,\n );\n for (const group of groups) {\n const groupMembers = [...group];\n if (!groupMembers.some((member) => selected.has(member))) continue;\n groupMembers.sort(\n (left, right) =>\n severityOrder[findings.get(left)!.severity.level] -\n severityOrder[findings.get(right)!.severity.level] ||\n (left < right ? -1 : left > right ? 1 : 0),\n );\n duplicateGroups.push(groupMembers);\n for (const member of groupMembers)\n canonical.set(member, groupMembers[0]!);\n }\n }\n return {\n uniqueFindingIds: [...new Set(ids.map((id) => canonical.get(id) ?? id))],\n duplicateGroups,\n deduplicationStatus: \"completed\",\n };\n }\n}\n"]}
|
|
@@ -4,13 +4,21 @@ export interface DeduplicateScanOptions {
|
|
|
4
4
|
workflowId?: string;
|
|
5
5
|
/** Findings API base URL. The scan's findings must already be indexed there. */
|
|
6
6
|
findingsUrl: string;
|
|
7
|
-
/** Search all repositories instead of the
|
|
7
|
+
/** Search all repositories instead of the scan's targetId. Defaults to false. */
|
|
8
8
|
allRepositories?: boolean;
|
|
9
9
|
signal?: AbortSignal;
|
|
10
10
|
}
|
|
11
|
+
export interface DeduplicateScanDirectoryOptions extends DeduplicateScanOptions {
|
|
12
|
+
/** Local repository checkout used to review duplicate candidates. */
|
|
13
|
+
repository: string;
|
|
14
|
+
/** Require the sealed artifacts to belong to this scan. */
|
|
15
|
+
expectedScanId?: string;
|
|
16
|
+
}
|
|
11
17
|
export interface DeduplicateScanResult extends DeduplicationResult {
|
|
12
18
|
scanId: string;
|
|
13
19
|
}
|
|
14
20
|
/** Review a saved scan against embedding candidates and persist accepted duplicate groups. */
|
|
15
21
|
export declare function deduplicateScan(scanId: string, options: DeduplicateScanOptions): Promise<DeduplicateScanResult>;
|
|
22
|
+
/** Review a complete, sealed scan directory without resolving local scan history. */
|
|
23
|
+
export declare function deduplicateScanDirectory(scanDirectory: string, options: DeduplicateScanDirectoryOptions): Promise<DeduplicateScanResult>;
|
|
16
24
|
//# sourceMappingURL=scan.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scan.d.ts","sourceRoot":"","sources":["../../src/deduplication/scan.ts"],"names":[],"mappings":"AAYA,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"scan.d.ts","sourceRoot":"","sources":["../../src/deduplication/scan.ts"],"names":[],"mappings":"AAYA,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,oBAAoB,CAAC;AAmB5B,MAAM,WAAW,sBAAsB;IACrC,8EAA8E;IAC9E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,WAAW,EAAE,MAAM,CAAC;IACpB,iFAAiF;IACjF,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,+BACf,SAAQ,sBAAsB;IAC9B,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,2DAA2D;IAC3D,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,qBAAsB,SAAQ,mBAAmB;IAChE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,8FAA8F;AAC9F,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,qBAAqB,CAAC,CAEhC;AAED,qFAAqF;AACrF,wBAAsB,wBAAwB,CAC5C,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,+BAA+B,GACvC,OAAO,CAAC,qBAAqB,CAAC,CAEhC"}
|
|
@@ -8,10 +8,21 @@ import { FindingsClient } from "../findings-client.js";
|
|
|
8
8
|
import { FindingWorkflow, workflowDestination, workflowDigest, } from "../finding-workflow.js";
|
|
9
9
|
import { publishScanToCustomInternal } from "../custom-publish.js";
|
|
10
10
|
import { CheckpointedReviewRunner, reviewSettingsDigest, } from "./checkpointed-review.js";
|
|
11
|
+
import { normalizeRepository } from "../targets.js";
|
|
11
12
|
/** Review a saved scan against embedding candidates and persist accepted duplicate groups. */
|
|
12
13
|
export async function deduplicateScan(scanId, options) {
|
|
13
14
|
return await deduplicateScanInternal(scanId, options);
|
|
14
15
|
}
|
|
16
|
+
/** Review a complete, sealed scan directory without resolving local scan history. */
|
|
17
|
+
export async function deduplicateScanDirectory(scanDirectory, options) {
|
|
18
|
+
return await deduplicateScanDirectoryInternal(scanDirectory, options);
|
|
19
|
+
}
|
|
20
|
+
/** @internal */
|
|
21
|
+
export async function deduplicateScanDirectoryInternal(scanDirectory, options, dependencies = {}) {
|
|
22
|
+
options.signal?.throwIfAborted();
|
|
23
|
+
const repository = await normalizeRepository(options.repository, options.signal);
|
|
24
|
+
return await deduplicateResolvedScan(scanDirectory, repository, options.expectedScanId, options, dependencies, await bundledPluginRoot(), true);
|
|
25
|
+
}
|
|
15
26
|
/** @internal */
|
|
16
27
|
export async function deduplicateScanInternal(scanId, options, dependencies = {}) {
|
|
17
28
|
options.signal?.throwIfAborted();
|
|
@@ -36,11 +47,16 @@ export async function deduplicateScanInternal(scanId, options, dependencies = {}
|
|
|
36
47
|
}, args);
|
|
37
48
|
}),
|
|
38
49
|
});
|
|
39
|
-
|
|
50
|
+
return await deduplicateResolvedScan(scan.scanDir, scan["targetPath"], scan.scanId, options, dependencies, pluginRoot, false);
|
|
51
|
+
}
|
|
52
|
+
async function deduplicateResolvedScan(selectedDirectory, repositoryPath, expectedScanId, options, dependencies, pluginRoot, bindRepository) {
|
|
53
|
+
const environment = dependencies.environment ?? process.env;
|
|
54
|
+
const { contract, scanDirectory } = await loadContractWithScanDirectory(selectedDirectory, {
|
|
40
55
|
pluginRoot,
|
|
41
|
-
expectedScanId
|
|
56
|
+
expectedScanId,
|
|
42
57
|
signal: options.signal,
|
|
43
58
|
});
|
|
59
|
+
const scanId = contract.manifest.scan.id;
|
|
44
60
|
const client = new FindingsClient(options.findingsUrl, options.signal, dependencies.fetch);
|
|
45
61
|
const scope = options.allRepositories === true
|
|
46
62
|
? { allRepositories: true }
|
|
@@ -53,7 +69,8 @@ export async function deduplicateScanInternal(scanId, options, dependencies = {}
|
|
|
53
69
|
if (workflow) {
|
|
54
70
|
await workflow.protectArtifacts(scanDirectory);
|
|
55
71
|
await workflow.bind({
|
|
56
|
-
|
|
72
|
+
...(bindRepository ? { repositoryPath } : {}),
|
|
73
|
+
scanId,
|
|
57
74
|
scanDir: scanDirectory,
|
|
58
75
|
artifactDigest: workflowDigest(contract),
|
|
59
76
|
destination: workflowDestination(options.findingsUrl),
|
|
@@ -63,7 +80,7 @@ export async function deduplicateScanInternal(scanId, options, dependencies = {}
|
|
|
63
80
|
await publishScanToCustomInternal(scanDirectory, {
|
|
64
81
|
findingsUrl: options.findingsUrl,
|
|
65
82
|
workflowId: options.workflowId,
|
|
66
|
-
expectedScanId:
|
|
83
|
+
expectedScanId: scanId,
|
|
67
84
|
signal: options.signal,
|
|
68
85
|
}, {
|
|
69
86
|
environment,
|
|
@@ -80,9 +97,9 @@ export async function deduplicateScanInternal(scanId, options, dependencies = {}
|
|
|
80
97
|
return saved.result;
|
|
81
98
|
}
|
|
82
99
|
const runner = dependencies.reviewRunner ??
|
|
83
|
-
new CodexReviewRunner(environment, undefined, options.signal,
|
|
100
|
+
new CodexReviewRunner(environment, undefined, options.signal, repositoryPath);
|
|
84
101
|
const checkpoints = workflow
|
|
85
|
-
? new CheckpointedReviewRunner(workflow, runner, await workflow.sourceSnapshot(
|
|
102
|
+
? new CheckpointedReviewRunner(workflow, runner, await workflow.sourceSnapshot(repositoryPath), scope, await reviewSettingsDigest(environment))
|
|
86
103
|
: undefined;
|
|
87
104
|
const deduplicator = new FindingDeduplicator({
|
|
88
105
|
potentialDuplicates: (findingId) => client.potentialDuplicates(findingId, scope),
|
|
@@ -91,7 +108,7 @@ export async function deduplicateScanInternal(scanId, options, dependencies = {}
|
|
|
91
108
|
const reviewed = await deduplicator.run(contract.findings.findings.map((finding) => finding.findingId));
|
|
92
109
|
await checkpoints?.assertSourceUnchanged();
|
|
93
110
|
options.signal?.throwIfAborted();
|
|
94
|
-
const result = { scanId
|
|
111
|
+
const result = { scanId, ...reviewed };
|
|
95
112
|
await workflow?.prepareDedupe(result, { groups: result.duplicateGroups });
|
|
96
113
|
await client.storeDedupeGroups(result.duplicateGroups);
|
|
97
114
|
return result;
|