@goodandready/dsh-moa 0.2.79 → 0.2.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +96 -3
- package/lib/events.js +2 -0
- package/lib/moa-schema.js +12 -0
- package/lib/pipeline/embedding-clustering.js +456 -0
- package/lib/pipeline/rebuttal-debate.js +335 -0
- package/lib/pipeline/runner.js +60 -3
- package/package.json +1 -1
package/lib/client.js
CHANGED
|
@@ -249,7 +249,14 @@ window.__ModuleLoader__.load({
|
|
|
249
249
|
"aggregator.hard_budget_limit_label": "Max Cost Per Turn (USD):",
|
|
250
250
|
"aggregator.hard_budget_policy_label": "Exceed Policy:",
|
|
251
251
|
"aggregator.hard_budget_policy_downgrade": "Auto-Downgrade to Local/Economic",
|
|
252
|
-
"aggregator.hard_budget_policy_fail": "Fail-Closed (Halt Execution)"
|
|
252
|
+
"aggregator.hard_budget_policy_fail": "Fail-Closed (Halt Execution)",
|
|
253
|
+
"aggregator.rebuttal_debate_title": "\u{1F5E3}\uFE0F Multi-Round Rebuttal Debate",
|
|
254
|
+
"aggregator.rebuttal_debate_desc": "Candidates cross-examine peer solutions and expose flaws before judge arbitration.",
|
|
255
|
+
"aggregator.rebuttal_rounds_label": "Debate Rounds:",
|
|
256
|
+
"aggregator.embedding_clustering_title": "\u{1F9ED} Blind Embedding Clustering",
|
|
257
|
+
"aggregator.embedding_clustering_desc": "Bypass LLM Judge when candidate embeddings converge into a dominant cluster (>= threshold).",
|
|
258
|
+
"aggregator.embedding_threshold_label": "Consensus Similarity Threshold:",
|
|
259
|
+
"aggregator.fallback_to_judge_label": "Fallback to Judge if Opinion Splits"
|
|
253
260
|
};
|
|
254
261
|
const zh = {
|
|
255
262
|
title: "\u667A\u80FD\u4F53\u6DF7\u5408 (MoA)",
|
|
@@ -489,7 +496,14 @@ window.__ModuleLoader__.load({
|
|
|
489
496
|
"aggregator.hard_budget_limit_label": "\u5355\u8F6E\u9884\u7B97\u4E0A\u9650 (USD):",
|
|
490
497
|
"aggregator.hard_budget_policy_label": "\u8D85\u989D\u7B56\u7565:",
|
|
491
498
|
"aggregator.hard_budget_policy_downgrade": "\u81EA\u52A8\u964D\u7EA7\u81F3\u672C\u5730/\u7ECF\u6D4E\u6A21\u578B",
|
|
492
|
-
"aggregator.hard_budget_policy_fail": "\u5B89\u5168\u7194\u65AD (\u505C\u6B62\u6267\u884C)"
|
|
499
|
+
"aggregator.hard_budget_policy_fail": "\u5B89\u5168\u7194\u65AD (\u505C\u6B62\u6267\u884C)",
|
|
500
|
+
"aggregator.rebuttal_debate_title": "\u{1F5E3}\uFE0F \u591A\u8F6E\u53CD\u9A73\u8FA9\u8BBA\u673A\u5236",
|
|
501
|
+
"aggregator.rebuttal_debate_desc": "\u5019\u9009\u6A21\u578B\u5728\u88C1\u5224\u4EF2\u88C1\u524D\u76F8\u4E92\u4EA4\u53C9\u5BA1\u67E5\u5E76\u6307\u51FA\u5BF9\u65B9\u6F0F\u6D1E\u4E0E\u7F3A\u9677\u3002",
|
|
502
|
+
"aggregator.rebuttal_rounds_label": "\u8FA9\u8BBA\u8F6E\u6570:",
|
|
503
|
+
"aggregator.embedding_clustering_title": "\u{1F9ED} \u76F2\u5411\u91CF\u5D4C\u5165\u805A\u7C7B",
|
|
504
|
+
"aggregator.embedding_clustering_desc": "\u5F53\u5019\u9009\u65B9\u6848\u7684\u5411\u91CF\u5D4C\u5165\u9AD8\u5EA6\u6536\u655B\u5E76\u5F62\u6210\u4E3B\u5BFC\u805A\u7C7B\u65F6\uFF0C\u7ED5\u8FC7LLM\u88C1\u5224\u76F4\u63A5\u4EA7\u51FA\u8D28\u5FC3\u5019\u9009\u3002",
|
|
505
|
+
"aggregator.embedding_threshold_label": "\u5171\u8BC6\u76F8\u4F3C\u5EA6\u9608\u503C:",
|
|
506
|
+
"aggregator.fallback_to_judge_label": "\u610F\u89C1\u5206\u6B67\u65F6\u56DE\u9000\u81F3\u88C1\u5224"
|
|
493
507
|
};
|
|
494
508
|
function makeT(dict, fallback) {
|
|
495
509
|
return function t(key, vars) {
|
|
@@ -2404,7 +2418,6 @@ window.__ModuleLoader__.load({
|
|
|
2404
2418
|
)
|
|
2405
2419
|
)
|
|
2406
2420
|
),
|
|
2407
|
-
// --- src/client/86-card-aggregator-ext2.js ---
|
|
2408
2421
|
/* Pre-Judge Local Syntax Gate (#303) & Early Semantic Consensus (#299) */
|
|
2409
2422
|
React.createElement(
|
|
2410
2423
|
"div",
|
|
@@ -2442,6 +2455,86 @@ window.__ModuleLoader__.load({
|
|
|
2442
2455
|
t("aggregator.early_consensus_desc") || "Bypass the expensive judge round when reference models converge on equivalent code solutions."
|
|
2443
2456
|
)
|
|
2444
2457
|
),
|
|
2458
|
+
/* Multi-Round Rebuttal Debate (#298) & Blind Embedding Clustering (#312) */
|
|
2459
|
+
React.createElement(
|
|
2460
|
+
"div",
|
|
2461
|
+
{ style: { display: "flex", flexDirection: "column", gap: 6, padding: "8px 12px", background: "var(--dsw-alias-bg-layer-2)", borderRadius: 8, border: "1px solid var(--dsw-alias-border-l2)", marginTop: 8 } },
|
|
2462
|
+
React.createElement(
|
|
2463
|
+
"label",
|
|
2464
|
+
{ style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer", fontSize: 13, fontWeight: 500 } },
|
|
2465
|
+
React.createElement("input", {
|
|
2466
|
+
type: "checkbox",
|
|
2467
|
+
checked: Boolean(currentPreset.rebuttal_debate_enabled),
|
|
2468
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, rebuttal_debate_enabled: e.target.checked })),
|
|
2469
|
+
style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
|
|
2470
|
+
}),
|
|
2471
|
+
t("aggregator.rebuttal_debate_title") || "\u{1F5E3}\uFE0F Multi-Round Rebuttal Debate"
|
|
2472
|
+
),
|
|
2473
|
+
React.createElement(
|
|
2474
|
+
"div",
|
|
2475
|
+
{ style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
|
|
2476
|
+
t("aggregator.rebuttal_debate_desc") || "Candidates cross-examine peer solutions and expose flaws before judge arbitration."
|
|
2477
|
+
),
|
|
2478
|
+
Boolean(currentPreset.rebuttal_debate_enabled) && React.createElement(
|
|
2479
|
+
"div",
|
|
2480
|
+
{ style: { display: "flex", alignItems: "center", gap: 8, marginLeft: 22, marginTop: 4 } },
|
|
2481
|
+
React.createElement("span", { style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)" } }, t("aggregator.rebuttal_rounds_label") || "Debate Rounds:"),
|
|
2482
|
+
React.createElement("input", {
|
|
2483
|
+
type: "number",
|
|
2484
|
+
min: 1,
|
|
2485
|
+
max: 3,
|
|
2486
|
+
value: currentPreset.rebuttal_rounds || 1,
|
|
2487
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, rebuttal_rounds: Math.min(3, Math.max(1, Number(e.target.value) || 1)) })),
|
|
2488
|
+
style: { width: 50, padding: "2px 6px", fontSize: 12, borderRadius: 4, border: "1px solid var(--dsw-alias-border-l2)", background: "var(--dsw-alias-bg-layer-1)" }
|
|
2489
|
+
})
|
|
2490
|
+
),
|
|
2491
|
+
React.createElement(
|
|
2492
|
+
"label",
|
|
2493
|
+
{ style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer", fontSize: 13, fontWeight: 500, marginTop: 6 } },
|
|
2494
|
+
React.createElement("input", {
|
|
2495
|
+
type: "checkbox",
|
|
2496
|
+
checked: Boolean(currentPreset.embedding_clustering_enabled),
|
|
2497
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, embedding_clustering_enabled: e.target.checked })),
|
|
2498
|
+
style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
|
|
2499
|
+
}),
|
|
2500
|
+
t("aggregator.embedding_clustering_title") || "\u{1F9ED} Blind Embedding Clustering"
|
|
2501
|
+
),
|
|
2502
|
+
React.createElement(
|
|
2503
|
+
"div",
|
|
2504
|
+
{ style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
|
|
2505
|
+
t("aggregator.embedding_clustering_desc") || "Bypass LLM Judge when candidate embeddings converge into a dominant cluster (>= threshold)."
|
|
2506
|
+
),
|
|
2507
|
+
Boolean(currentPreset.embedding_clustering_enabled) && React.createElement(
|
|
2508
|
+
"div",
|
|
2509
|
+
{ style: { display: "flex", flexDirection: "column", gap: 4, marginLeft: 22, marginTop: 4 } },
|
|
2510
|
+
React.createElement(
|
|
2511
|
+
"div",
|
|
2512
|
+
{ style: { display: "flex", alignItems: "center", gap: 8 } },
|
|
2513
|
+
React.createElement("span", { style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)" } }, t("aggregator.embedding_threshold_label") || "Consensus Similarity Threshold:"),
|
|
2514
|
+
React.createElement("input", {
|
|
2515
|
+
type: "number",
|
|
2516
|
+
step: 0.01,
|
|
2517
|
+
min: 0.5,
|
|
2518
|
+
max: 0.99,
|
|
2519
|
+
value: currentPreset.embedding_consensus_threshold ?? 0.88,
|
|
2520
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, embedding_consensus_threshold: Number(e.target.value) || 0.88 })),
|
|
2521
|
+
style: { width: 60, padding: "2px 6px", fontSize: 12, borderRadius: 4, border: "1px solid var(--dsw-alias-border-l2)", background: "var(--dsw-alias-bg-layer-1)" }
|
|
2522
|
+
})
|
|
2523
|
+
),
|
|
2524
|
+
React.createElement(
|
|
2525
|
+
"label",
|
|
2526
|
+
{ style: { display: "flex", alignItems: "center", gap: 6, cursor: "pointer", fontSize: 12 } },
|
|
2527
|
+
React.createElement("input", {
|
|
2528
|
+
type: "checkbox",
|
|
2529
|
+
checked: currentPreset.fallback_to_judge_on_split !== false,
|
|
2530
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, fallback_to_judge_on_split: e.target.checked })),
|
|
2531
|
+
style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))" }
|
|
2532
|
+
}),
|
|
2533
|
+
t("aggregator.fallback_to_judge_label") || "Fallback to Judge if Opinion Splits"
|
|
2534
|
+
)
|
|
2535
|
+
)
|
|
2536
|
+
),
|
|
2537
|
+
// --- src/client/86-card-aggregator-ext2.js ---
|
|
2445
2538
|
/* Multi-Turn Context & Diff Pruning (#176) */
|
|
2446
2539
|
React.createElement(
|
|
2447
2540
|
"div",
|
package/lib/events.js
CHANGED
|
@@ -22,6 +22,8 @@ export const MOA_EVENTS = {
|
|
|
22
22
|
CRITIQUE_COMPLETE: 'moa/critique-complete',
|
|
23
23
|
JUDGE_VERDICT: 'moa/judge-verdict',
|
|
24
24
|
CONTEXT_PRUNING: 'moa/context-pruning',
|
|
25
|
+
DEBATE_ROUND: 'moa/debate-round',
|
|
26
|
+
EMBEDDING_CLUSTER: 'moa/embedding-cluster',
|
|
25
27
|
CACHE_HIT: 'moa/cache-hit',
|
|
26
28
|
RUN_FINISH: 'moa/run-finish',
|
|
27
29
|
}
|
package/lib/moa-schema.js
CHANGED
|
@@ -112,6 +112,12 @@ export const PresetSchema = z.object({
|
|
|
112
112
|
hard_budget_guard_enabled: z.boolean().default(false),
|
|
113
113
|
hard_budget_limit_usd: z.number().default(0.02),
|
|
114
114
|
hard_budget_policy: z.string().default('auto_downgrade'),
|
|
115
|
+
rebuttal_debate_enabled: z.boolean().default(false),
|
|
116
|
+
rebuttal_rounds: z.number().default(1),
|
|
117
|
+
rebuttal_max_critique_tokens: z.number().default(1024),
|
|
118
|
+
embedding_clustering_enabled: z.boolean().default(false),
|
|
119
|
+
embedding_consensus_threshold: z.number().default(0.88),
|
|
120
|
+
fallback_to_judge_on_split: z.boolean().default(true),
|
|
115
121
|
})
|
|
116
122
|
|
|
117
123
|
export const Config = z.object({
|
|
@@ -170,6 +176,12 @@ export const Config = z.object({
|
|
|
170
176
|
hard_budget_guard_enabled: z.boolean().default(false).volatile(),
|
|
171
177
|
hard_budget_limit_usd: z.number().default(0.02).volatile(),
|
|
172
178
|
hard_budget_policy: z.string().default('auto_downgrade').volatile(),
|
|
179
|
+
rebuttal_debate_enabled: z.boolean().default(false).volatile(),
|
|
180
|
+
rebuttal_rounds: z.number().default(1).volatile(),
|
|
181
|
+
rebuttal_max_critique_tokens: z.number().default(1024).volatile(),
|
|
182
|
+
embedding_clustering_enabled: z.boolean().default(false).volatile(),
|
|
183
|
+
embedding_consensus_threshold: z.number().default(0.88).volatile(),
|
|
184
|
+
fallback_to_judge_on_split: z.boolean().default(true).volatile(),
|
|
173
185
|
smart_routing_model: ModelSlotSchema.default({ provider: '', model: '' }).volatile(),
|
|
174
186
|
prices: z.dict(PriceRow).default({}).volatile(),
|
|
175
187
|
presets: z.array(PresetSchema).default([
|
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Blind Embedding Clustering for Zero-LLM Fast Consensus (#312)
|
|
3
|
+
*
|
|
4
|
+
* Clusters candidate responses using local high-dimensional n-gram semantic embeddings.
|
|
5
|
+
* When a dominant cluster exhibits high semantic convergence (cosine similarity >= threshold),
|
|
6
|
+
* the plugin automatically promotes the cluster centroid candidate without invoking
|
|
7
|
+
* the expensive LLM Judge, saving significant tokens and latency.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { logger } from '../logger.js'
|
|
11
|
+
import { promoteCandidateWorkspace, cleanMoaWorkspaces } from '../workspace/fs.js'
|
|
12
|
+
import { createPromotedPreview } from '../live-canvas.js'
|
|
13
|
+
import { recordMoaRunAsync, candidatesForHistory } from '../history.js'
|
|
14
|
+
import { emitEvent, MOA_EVENTS } from '../events.js'
|
|
15
|
+
import { estimateTokenCost } from '../pricing.js'
|
|
16
|
+
|
|
17
|
+
const EMBEDDING_DIMENSIONS = 1024
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* 32-bit FNV-1a hash function for strings
|
|
21
|
+
* @param {string} str
|
|
22
|
+
* @returns {number}
|
|
23
|
+
*/
|
|
24
|
+
export function fnv1a32(str) {
|
|
25
|
+
let hash = 2166136261
|
|
26
|
+
for (let i = 0; i < str.length; i++) {
|
|
27
|
+
hash ^= str.charCodeAt(i)
|
|
28
|
+
hash = Math.imul(hash, 16777619)
|
|
29
|
+
}
|
|
30
|
+
return hash >>> 0
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Normalizes text/code for embedding extraction.
|
|
35
|
+
* @param {string} text
|
|
36
|
+
* @returns {string}
|
|
37
|
+
*/
|
|
38
|
+
export function normalizeText(text = '') {
|
|
39
|
+
if (typeof text !== 'string') return ''
|
|
40
|
+
return text
|
|
41
|
+
.replace(/\/\*[\s\S]*?\*\//g, '')
|
|
42
|
+
.replace(/\/\/.*$/gm, '')
|
|
43
|
+
.replace(/(^|\s)#.*$/gm, '$1')
|
|
44
|
+
.replace(/\r?\n/g, ' ')
|
|
45
|
+
.replace(/[ \t]+/g, ' ')
|
|
46
|
+
.trim()
|
|
47
|
+
.toLowerCase()
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Generates a unit-normalized Float32Array embedding vector from text
|
|
52
|
+
* using word tokens, code operators, and character n-grams.
|
|
53
|
+
* @param {string} text
|
|
54
|
+
* @param {number} [dimensions=1024]
|
|
55
|
+
* @returns {Float32Array}
|
|
56
|
+
*/
|
|
57
|
+
export function generateEmbedding(text = '', dimensions = EMBEDDING_DIMENSIONS) {
|
|
58
|
+
const vector = new Float32Array(dimensions)
|
|
59
|
+
const normalized = normalizeText(text)
|
|
60
|
+
if (!normalized) return vector
|
|
61
|
+
|
|
62
|
+
// 1. Tokens including keywords, identifiers, and syntax operators
|
|
63
|
+
const tokens = normalized.match(/[a-z0-9_$]+|===|!==|<=|>=|=>|\+\+|--|&&|\|\||\.\.\.|[{}()[\];,+\-*\/%=<>!&|^~?:.]/g) || []
|
|
64
|
+
const counts = new Map()
|
|
65
|
+
|
|
66
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
67
|
+
const t = tokens[i]
|
|
68
|
+
const h = fnv1a32(t) % dimensions
|
|
69
|
+
counts.set(h, (counts.get(h) || 0) + 1.2)
|
|
70
|
+
|
|
71
|
+
// Token bigrams
|
|
72
|
+
if (i < tokens.length - 1) {
|
|
73
|
+
const bigram = `${t}_${tokens[i + 1]}`
|
|
74
|
+
const h2 = fnv1a32(bigram) % dimensions
|
|
75
|
+
counts.set(h2, (counts.get(h2) || 0) + 0.8)
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// 2. Character 3-grams for morphology and syntax
|
|
80
|
+
const charSpan = Math.min(normalized.length, 12000)
|
|
81
|
+
for (let i = 0; i <= charSpan - 3; i++) {
|
|
82
|
+
const trigram = normalized.slice(i, i + 3)
|
|
83
|
+
const h3 = fnv1a32(trigram) % dimensions
|
|
84
|
+
counts.set(h3, (counts.get(h3) || 0) + 0.3)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// 3. Sublinear TF weighting: 1 + ln(count)
|
|
88
|
+
let sumSq = 0
|
|
89
|
+
for (const [bucket, count] of counts.entries()) {
|
|
90
|
+
const val = 1 + Math.log(count)
|
|
91
|
+
vector[bucket] = val
|
|
92
|
+
sumSq += val * val
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// 4. L2 Normalization
|
|
96
|
+
if (sumSq > 0) {
|
|
97
|
+
const norm = Math.sqrt(sumSq)
|
|
98
|
+
for (let i = 0; i < dimensions; i++) {
|
|
99
|
+
vector[i] /= norm
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
return vector
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Computes cosine similarity between two unit-normalized vectors.
|
|
108
|
+
* Dot product equals cosine similarity for unit vectors.
|
|
109
|
+
* @param {Float32Array} vecA
|
|
110
|
+
* @param {Float32Array} vecB
|
|
111
|
+
* @returns {number}
|
|
112
|
+
*/
|
|
113
|
+
export function computeCosineSimilarity(vecA, vecB) {
|
|
114
|
+
if (!vecA || !vecB || vecA.length !== vecB.length) return 0.0
|
|
115
|
+
let dot = 0
|
|
116
|
+
for (let i = 0; i < vecA.length; i++) {
|
|
117
|
+
dot += vecA[i] * vecB[i]
|
|
118
|
+
}
|
|
119
|
+
const clamped = Math.max(0.0, Math.min(1.0, dot))
|
|
120
|
+
return Number(clamped.toFixed(4))
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Builds pairwise cosine similarity matrix for candidate solutions.
|
|
125
|
+
* @param {Array<object>} candidates
|
|
126
|
+
* @returns {{ vectors: Float32Array[], matrix: number[][] }}
|
|
127
|
+
*/
|
|
128
|
+
export function buildSimilarityMatrix(candidates = []) {
|
|
129
|
+
const n = candidates.length
|
|
130
|
+
const vectors = candidates.map((c) => generateEmbedding(c.text || ''))
|
|
131
|
+
const matrix = Array.from({ length: n }, () => new Array(n).fill(0))
|
|
132
|
+
|
|
133
|
+
for (let i = 0; i < n; i++) {
|
|
134
|
+
matrix[i][i] = 1.0
|
|
135
|
+
for (let j = i + 1; j < n; j++) {
|
|
136
|
+
const sim = computeCosineSimilarity(vectors[i], vectors[j])
|
|
137
|
+
matrix[i][j] = sim
|
|
138
|
+
matrix[j][i] = sim
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
return { vectors, matrix }
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Clusters candidate embeddings and determines if consensus was reached.
|
|
147
|
+
* @param {Array<object>} candidates
|
|
148
|
+
* @param {object} [options={}]
|
|
149
|
+
* @returns {object}
|
|
150
|
+
*/
|
|
151
|
+
export function clusterCandidateEmbeddings(candidates = [], options = {}) {
|
|
152
|
+
const threshold = typeof options.threshold === 'number' ? options.threshold : 0.88
|
|
153
|
+
const fallbackToJudgeOnSplit = options.fallbackToJudgeOnSplit !== false
|
|
154
|
+
const minMajorityRatio = typeof options.minMajorityRatio === 'number' ? options.minMajorityRatio : 0.60
|
|
155
|
+
|
|
156
|
+
if (!Array.isArray(candidates) || candidates.length < 2) {
|
|
157
|
+
return {
|
|
158
|
+
consensusReached: false,
|
|
159
|
+
threshold,
|
|
160
|
+
clusters: [],
|
|
161
|
+
dominantCluster: null,
|
|
162
|
+
centroidCandidate: null,
|
|
163
|
+
centroidIndex: 0,
|
|
164
|
+
similarityMatrix: [],
|
|
165
|
+
fallbackToJudge: fallbackToJudgeOnSplit,
|
|
166
|
+
reason: 'Insufficient candidates (need at least 2)',
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const n = candidates.length
|
|
171
|
+
const { vectors, matrix } = buildSimilarityMatrix(candidates)
|
|
172
|
+
|
|
173
|
+
// Threshold clustering: connect candidates with pairwise similarity >= threshold
|
|
174
|
+
const clusters = []
|
|
175
|
+
const assigned = new Set()
|
|
176
|
+
|
|
177
|
+
for (let i = 0; i < n; i++) {
|
|
178
|
+
if (assigned.has(i)) continue
|
|
179
|
+
const clusterMembers = [i]
|
|
180
|
+
assigned.add(i)
|
|
181
|
+
|
|
182
|
+
// Find all peers reachable with similarity >= threshold
|
|
183
|
+
for (let j = 0; j < n; j++) {
|
|
184
|
+
if (assigned.has(j)) continue
|
|
185
|
+
// Qualifies if similarity to cluster seed or average to cluster >= threshold
|
|
186
|
+
const simToSeed = matrix[i][j]
|
|
187
|
+
const avgSim = clusterMembers.reduce((sum, idx) => sum + matrix[j][idx], 0) / clusterMembers.length
|
|
188
|
+
if (simToSeed >= threshold || avgSim >= threshold) {
|
|
189
|
+
clusterMembers.push(j)
|
|
190
|
+
assigned.add(j)
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
clusters.push(clusterMembers)
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Find dominant cluster (maximum members)
|
|
197
|
+
clusters.sort((a, b) => b.length - a.length)
|
|
198
|
+
const largestCluster = clusters[0] || []
|
|
199
|
+
|
|
200
|
+
// Compute intra-cluster average similarity
|
|
201
|
+
let intraClusterSimSum = 0
|
|
202
|
+
let pairCount = 0
|
|
203
|
+
for (let a = 0; a < largestCluster.length; a++) {
|
|
204
|
+
for (let b = a + 1; b < largestCluster.length; b++) {
|
|
205
|
+
intraClusterSimSum += matrix[largestCluster[a]][largestCluster[b]]
|
|
206
|
+
pairCount++
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
const avgIntraSimilarity = pairCount > 0 ? Number((intraClusterSimSum / pairCount).toFixed(4)) : 1.0
|
|
210
|
+
|
|
211
|
+
// Majority condition: cluster has >= required members and high internal similarity
|
|
212
|
+
const requiredCount = Math.max(2, Math.ceil(n * minMajorityRatio))
|
|
213
|
+
const consensusReached = largestCluster.length >= requiredCount && (avgIntraSimilarity >= threshold || largestCluster.length === n)
|
|
214
|
+
|
|
215
|
+
let centroidIndex = -1
|
|
216
|
+
let centroidCandidate = null
|
|
217
|
+
|
|
218
|
+
if (consensusReached) {
|
|
219
|
+
// Select centroid: member with highest average similarity to peers in the dominant cluster
|
|
220
|
+
let bestAvgSim = -1
|
|
221
|
+
for (const idx of largestCluster) {
|
|
222
|
+
let simTotal = 0
|
|
223
|
+
for (const peerIdx of largestCluster) {
|
|
224
|
+
if (idx !== peerIdx) simTotal += matrix[idx][peerIdx]
|
|
225
|
+
}
|
|
226
|
+
const candidateAvg = largestCluster.length > 1 ? simTotal / (largestCluster.length - 1) : 1.0
|
|
227
|
+
|
|
228
|
+
// Favor syntactically valid candidates in tie/near-tie situations
|
|
229
|
+
const hasSyntaxError = candidates[idx].syntaxValidation?.valid === false
|
|
230
|
+
const adjustedScore = candidateAvg - (hasSyntaxError ? 0.05 : 0)
|
|
231
|
+
|
|
232
|
+
if (adjustedScore > bestAvgSim) {
|
|
233
|
+
bestAvgSim = adjustedScore
|
|
234
|
+
centroidIndex = idx
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
centroidCandidate = candidates[centroidIndex] || null
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const dominantClusterInfo = {
|
|
242
|
+
size: largestCluster.length,
|
|
243
|
+
memberIndices: largestCluster.map((i) => candidates[i].index ?? (i + 1)),
|
|
244
|
+
avgSimilarity: avgIntraSimilarity,
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
return {
|
|
248
|
+
consensusReached,
|
|
249
|
+
threshold,
|
|
250
|
+
dominantCluster: dominantClusterInfo,
|
|
251
|
+
clusters: clusters.map((c, id) => ({
|
|
252
|
+
id,
|
|
253
|
+
size: c.length,
|
|
254
|
+
indices: c.map((i) => candidates[i].index ?? (i + 1)),
|
|
255
|
+
})),
|
|
256
|
+
centroidCandidate,
|
|
257
|
+
centroidIndex: centroidCandidate ? (centroidCandidate.index ?? (centroidIndex + 1)) : 0,
|
|
258
|
+
similarityMatrix: matrix,
|
|
259
|
+
fallbackToJudge: !consensusReached && fallbackToJudgeOnSplit,
|
|
260
|
+
reason: consensusReached
|
|
261
|
+
? `Blind embedding consensus reached (${largestCluster.length}/${n} models converged with avg similarity ${avgIntraSimilarity} >= ${threshold})`
|
|
262
|
+
: `No dominant embedding cluster converged (largest has ${largestCluster.length}/${n} with avg sim ${avgIntraSimilarity} < ${threshold})`,
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Estimates saved tokens and USD cost from bypassing the judge round.
|
|
268
|
+
* @param {Array<object>} candidates
|
|
269
|
+
* @param {string} userPrompt
|
|
270
|
+
* @param {number} maxTokens
|
|
271
|
+
* @param {object} prices
|
|
272
|
+
* @returns {{ skippedTokens: number, skippedCostUsd: number }}
|
|
273
|
+
*/
|
|
274
|
+
export function estimateSkippedJudgeUsage(candidates = [], userPrompt = '', maxTokens = 4096, prices = {}) {
|
|
275
|
+
const candidateChars = candidates.reduce((acc, c) => acc + (c.text?.length || 0), 0)
|
|
276
|
+
const promptChars = userPrompt.length
|
|
277
|
+
const estimatedInputTokens = Math.round((candidateChars + promptChars) / 3.8)
|
|
278
|
+
const estimatedOutputTokens = Math.min(maxTokens, 1500)
|
|
279
|
+
const skippedTokens = estimatedInputTokens + estimatedOutputTokens
|
|
280
|
+
|
|
281
|
+
const costEst = estimateTokenCost({
|
|
282
|
+
provider: 'default',
|
|
283
|
+
model: 'judge',
|
|
284
|
+
inputTokens: estimatedInputTokens,
|
|
285
|
+
outputTokens: estimatedOutputTokens,
|
|
286
|
+
prices,
|
|
287
|
+
})
|
|
288
|
+
|
|
289
|
+
return {
|
|
290
|
+
skippedTokens,
|
|
291
|
+
skippedCostUsd: Number((costEst?.costUsd || 0.005).toFixed(5)),
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Evaluates candidate responses for blind embedding clustering and executes fast zero-LLM consensus if reached.
|
|
297
|
+
* @param {object} params
|
|
298
|
+
* @returns {Promise<{ ok: boolean, consensusReached: boolean, payload?: object, error?: Error, abort?: boolean }>}
|
|
299
|
+
*/
|
|
300
|
+
export async function stageEmbeddingClustering({
|
|
301
|
+
successfulRefs = [],
|
|
302
|
+
referenceOutputs = [],
|
|
303
|
+
preset = {},
|
|
304
|
+
userPrompt = '',
|
|
305
|
+
prices = {},
|
|
306
|
+
maxTokens = 4096,
|
|
307
|
+
cwd = process.cwd(),
|
|
308
|
+
runId = '',
|
|
309
|
+
signal = null,
|
|
310
|
+
promoteOpts = {},
|
|
311
|
+
allowCandidateOverride = false,
|
|
312
|
+
isRefinement = false,
|
|
313
|
+
historyFilePath = null,
|
|
314
|
+
startTime = Date.now(),
|
|
315
|
+
collectedCtx = null,
|
|
316
|
+
liveCanvas = null,
|
|
317
|
+
ctx = null,
|
|
318
|
+
}) {
|
|
319
|
+
const enabled = Boolean(preset?.embedding_clustering_enabled || preset?.embeddingClusteringEnabled)
|
|
320
|
+
const threshold = typeof (preset?.embedding_consensus_threshold ?? preset?.embeddingConsensusThreshold) === 'number'
|
|
321
|
+
? (preset.embedding_consensus_threshold ?? preset.embeddingConsensusThreshold)
|
|
322
|
+
: 0.88
|
|
323
|
+
const fallbackToJudgeOnSplit = (preset?.fallback_to_judge_on_split ?? preset?.fallbackToJudgeOnSplit) !== false
|
|
324
|
+
|
|
325
|
+
if (!enabled || successfulRefs.length < 2) {
|
|
326
|
+
return { ok: true, consensusReached: false }
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
const result = clusterCandidateEmbeddings(successfulRefs, {
|
|
330
|
+
threshold,
|
|
331
|
+
fallbackToJudgeOnSplit,
|
|
332
|
+
})
|
|
333
|
+
|
|
334
|
+
emitEvent(ctx, MOA_EVENTS.EMBEDDING_CLUSTER, {
|
|
335
|
+
runId,
|
|
336
|
+
consensusReached: result.consensusReached,
|
|
337
|
+
threshold,
|
|
338
|
+
dominantCluster: result.dominantCluster,
|
|
339
|
+
clusters: result.clusters,
|
|
340
|
+
centroidIndex: result.centroidIndex,
|
|
341
|
+
fallbackToJudge: result.fallbackToJudge,
|
|
342
|
+
reason: result.reason,
|
|
343
|
+
})
|
|
344
|
+
|
|
345
|
+
if (!result.consensusReached || !result.centroidCandidate) {
|
|
346
|
+
logger.info(`[dsh-moa] Embedding clustering: ${result.reason}. Proceeding to arbitration.`)
|
|
347
|
+
return { ok: true, consensusReached: false, fallbackToJudge: result.fallbackToJudge }
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
const winner = result.centroidCandidate
|
|
351
|
+
logger.info(`[dsh-moa] Blind embedding fast consensus reached! Winner centroid: candidate ${result.centroidIndex}`)
|
|
352
|
+
|
|
353
|
+
// Promote centroid candidate files to project workspace
|
|
354
|
+
let promotedFiles = []
|
|
355
|
+
let promoteErr = null
|
|
356
|
+
if (winner.files?.length > 0) {
|
|
357
|
+
try {
|
|
358
|
+
promotedFiles = await promoteCandidateWorkspace(cwd, winner.index, {
|
|
359
|
+
...promoteOpts,
|
|
360
|
+
runId,
|
|
361
|
+
keepMoa: allowCandidateOverride,
|
|
362
|
+
})
|
|
363
|
+
} catch (err) {
|
|
364
|
+
promoteErr = err
|
|
365
|
+
}
|
|
366
|
+
} else if (!allowCandidateOverride) {
|
|
367
|
+
await cleanMoaWorkspaces(cwd, { runId })
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
if (promoteErr) {
|
|
371
|
+
if (!allowCandidateOverride) await cleanMoaWorkspaces(cwd, { runId })
|
|
372
|
+
if (signal?.aborted || promoteErr?.message?.includes('aborted')) {
|
|
373
|
+
return { ok: false, consensusReached: true, abort: true, error: new Error('Turn aborted') }
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
const skippedUsage = estimateSkippedJudgeUsage(successfulRefs, userPrompt, maxTokens, prices)
|
|
378
|
+
const livePreview = await createPromotedPreview(liveCanvas, cwd, promotedFiles, referenceOutputs, {
|
|
379
|
+
winningIndex: winner.index,
|
|
380
|
+
})
|
|
381
|
+
|
|
382
|
+
const totalTokens = referenceOutputs.reduce((acc, r) => acc + (r.usage?.totalTokens || 0), 0)
|
|
383
|
+
const totalCostUsd = Number(referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0).toFixed(5))
|
|
384
|
+
const durationMs = Date.now() - startTime
|
|
385
|
+
|
|
386
|
+
try {
|
|
387
|
+
await recordMoaRunAsync({
|
|
388
|
+
id: runId,
|
|
389
|
+
cwd,
|
|
390
|
+
prompt: userPrompt,
|
|
391
|
+
preset: preset?.name || 'default',
|
|
392
|
+
isRefinement,
|
|
393
|
+
isEarlyConsensus: true,
|
|
394
|
+
isEmbeddingConsensus: true,
|
|
395
|
+
candidates: candidatesForHistory(referenceOutputs),
|
|
396
|
+
aggregator: 'Blind Embedding Consensus (Bypassed)',
|
|
397
|
+
winnerIndex: winner.index,
|
|
398
|
+
winnerModel: winner.label,
|
|
399
|
+
promotedFiles,
|
|
400
|
+
totalTokens,
|
|
401
|
+
totalCostUsd,
|
|
402
|
+
skippedJudgeTokens: skippedUsage.skippedTokens,
|
|
403
|
+
durationMs,
|
|
404
|
+
}, historyFilePath || undefined)
|
|
405
|
+
} catch (histErr) {
|
|
406
|
+
logger.warn('[dsh-moa] Failed to record embedding consensus in history:', histErr?.message || histErr)
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
emitEvent(ctx, MOA_EVENTS.RUN_FINISH, {
|
|
410
|
+
runId,
|
|
411
|
+
durationMs,
|
|
412
|
+
totalTokens,
|
|
413
|
+
totalCostUsd,
|
|
414
|
+
promotedFiles,
|
|
415
|
+
winnerIndex: winner.index,
|
|
416
|
+
isEarlyConsensus: true,
|
|
417
|
+
isEmbeddingConsensus: true,
|
|
418
|
+
})
|
|
419
|
+
|
|
420
|
+
return {
|
|
421
|
+
ok: true,
|
|
422
|
+
consensusReached: true,
|
|
423
|
+
payload: {
|
|
424
|
+
kind: 'synthesis',
|
|
425
|
+
content: winner.text,
|
|
426
|
+
aggregator: 'Blind Embedding Consensus (Bypassed)',
|
|
427
|
+
references: referenceOutputs,
|
|
428
|
+
presetName: preset?.name || 'default',
|
|
429
|
+
isRefinement,
|
|
430
|
+
isEarlyConsensus: true,
|
|
431
|
+
isEmbeddingConsensus: true,
|
|
432
|
+
winningIndex: winner.index,
|
|
433
|
+
winnerModel: winner.label,
|
|
434
|
+
promotedFiles,
|
|
435
|
+
runId,
|
|
436
|
+
embeddingInfo: {
|
|
437
|
+
dominantCluster: result.dominantCluster,
|
|
438
|
+
threshold,
|
|
439
|
+
clusters: result.clusters,
|
|
440
|
+
centroidIndex: result.centroidIndex,
|
|
441
|
+
},
|
|
442
|
+
...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
443
|
+
usage: {
|
|
444
|
+
totalTokens,
|
|
445
|
+
totalCostUsd,
|
|
446
|
+
skippedJudgeTokens: skippedUsage.skippedTokens,
|
|
447
|
+
skippedJudgeCostUsd: skippedUsage.skippedCostUsd,
|
|
448
|
+
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
449
|
+
aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 },
|
|
450
|
+
},
|
|
451
|
+
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
452
|
+
skippedList: collectedCtx?.skippedList || [],
|
|
453
|
+
durationMs,
|
|
454
|
+
},
|
|
455
|
+
}
|
|
456
|
+
}
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-Round Rebuttal Debate between Candidates (#298)
|
|
3
|
+
*
|
|
4
|
+
* Implements multi-round cross-examination and rebuttal discussion between candidate models
|
|
5
|
+
* before final arbitration by the Judge. Candidates anonymize and analyze peer solutions,
|
|
6
|
+
* identify flaws, security edge-cases, and hallucinations, and refine their own answers.
|
|
7
|
+
* The structured debate protocol is then passed to the Judge for weighted arbitration.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { logger } from '../logger.js'
|
|
11
|
+
import { callWithTransientRetry } from '../moa-candidates.js'
|
|
12
|
+
import { extractFileBlocks, writeCandidateWorkspace } from '../workspace/fs.js'
|
|
13
|
+
import { verifyFileSyntax } from '../workspace/diff.js'
|
|
14
|
+
import { estimateTokenCost } from '../pricing.js'
|
|
15
|
+
import { emitEvent, MOA_EVENTS } from '../events.js'
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Builds the rebuttal prompt for a candidate in a given debate round.
|
|
19
|
+
* @param {object} params
|
|
20
|
+
* @returns {string}
|
|
21
|
+
*/
|
|
22
|
+
export function buildRebuttalPrompt({
|
|
23
|
+
userPrompt,
|
|
24
|
+
candidateText,
|
|
25
|
+
opponentAnswers,
|
|
26
|
+
roundNumber = 1,
|
|
27
|
+
totalRounds = 1,
|
|
28
|
+
isBlind = true,
|
|
29
|
+
}) {
|
|
30
|
+
const opponentsFormatted = opponentAnswers
|
|
31
|
+
.map((op, idx) => {
|
|
32
|
+
const label = op.label || `Candidate ${idx + 1}`
|
|
33
|
+
const persona = op.role_persona ? ` [Persona: ${op.role_persona}]` : ''
|
|
34
|
+
return `### ⚔️ Peer Solution [${label}]${persona}:\n${op.text}`
|
|
35
|
+
})
|
|
36
|
+
.join('\n\n')
|
|
37
|
+
|
|
38
|
+
return `You are an expert software engineer participating in a rigorous Multi-Agent Consilium Rebuttal Debate (Round ${roundNumber} of ${totalRounds}).
|
|
39
|
+
Your objective is to stress-test peer solutions, expose bugs, logical vulnerabilities, and hallucinations, and defend and refine your own solution.
|
|
40
|
+
|
|
41
|
+
=== ORIGINAL USER REQUEST ===
|
|
42
|
+
${userPrompt}
|
|
43
|
+
|
|
44
|
+
=== YOUR PREVIOUS PROPOSAL ===
|
|
45
|
+
${candidateText}
|
|
46
|
+
|
|
47
|
+
=== ALTERNATIVE PEER SOLUTIONS ===
|
|
48
|
+
${opponentsFormatted}
|
|
49
|
+
|
|
50
|
+
=== REBUTTAL DEBATE INSTRUCTIONS ===
|
|
51
|
+
Analyze the peer solutions with uncompromising technical rigor.
|
|
52
|
+
Structure your response EXACTLY into these three labeled sections:
|
|
53
|
+
|
|
54
|
+
### CRITIQUE OF PEERS
|
|
55
|
+
For each peer candidate, cite concrete issues:
|
|
56
|
+
- Identify logic bugs, off-by-one errors, uncaught exceptions, or security holes.
|
|
57
|
+
- Highlight missing edge cases or unhandled failure states.
|
|
58
|
+
- Point out inaccurate assumptions or hallucinations.
|
|
59
|
+
|
|
60
|
+
### REBUTTAL & DEFENSE
|
|
61
|
+
- Address the potential weaknesses or tradeoffs in your own approach.
|
|
62
|
+
- Explain why your architectural or algorithmic choices are superior or more maintainable.
|
|
63
|
+
|
|
64
|
+
### REFINED SOLUTION
|
|
65
|
+
- Provide your complete, improved, and final solution incorporating valid insights from the debate.
|
|
66
|
+
- Output complete code without placeholders or omissions.`
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Parses structured rebuttal debate output from a model response.
|
|
71
|
+
* @param {string} rawText
|
|
72
|
+
* @returns {{ critiqueText: string, defenseText: string, refinedText: string }}
|
|
73
|
+
*/
|
|
74
|
+
export function parseRebuttalResponse(rawText = '') {
|
|
75
|
+
if (typeof rawText !== 'string') {
|
|
76
|
+
return { critiqueText: '', defenseText: '', refinedText: '' }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
let critiqueText = ''
|
|
80
|
+
let defenseText = ''
|
|
81
|
+
let refinedText = rawText
|
|
82
|
+
|
|
83
|
+
const critiqueMatch = rawText.match(/### CRITIQUE OF PEERS\s*([\s\S]*?)(?=### REBUTTAL & DEFENSE|### REFINED SOLUTION|$)/i)
|
|
84
|
+
if (critiqueMatch) {
|
|
85
|
+
critiqueText = critiqueMatch[1].trim()
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const defenseMatch = rawText.match(/### REBUTTAL & DEFENSE\s*([\s\S]*?)(?=### REFINED SOLUTION|$)/i)
|
|
89
|
+
if (defenseMatch) {
|
|
90
|
+
defenseText = defenseMatch[1].trim()
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const refinedMatch = rawText.match(/### REFINED SOLUTION\s*([\s\S]*)$/i)
|
|
94
|
+
if (refinedMatch && refinedMatch[1].trim().length > 50) {
|
|
95
|
+
refinedText = refinedMatch[1].trim()
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return {
|
|
99
|
+
critiqueText,
|
|
100
|
+
defenseText,
|
|
101
|
+
refinedText,
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Executes a single round of rebuttal debate across all active candidates.
|
|
107
|
+
* @param {object} params
|
|
108
|
+
* @returns {Promise<{ round: number, critiques: Array<object>, updatedCandidates: Array<object> }>}
|
|
109
|
+
*/
|
|
110
|
+
export async function executeDebateRound({
|
|
111
|
+
candidates = [],
|
|
112
|
+
userPrompt = '',
|
|
113
|
+
roundNumber = 1,
|
|
114
|
+
totalRounds = 1,
|
|
115
|
+
isBlindEvaluation = true,
|
|
116
|
+
callLlm,
|
|
117
|
+
refTemp = 0.5,
|
|
118
|
+
maxTokens = 4096,
|
|
119
|
+
refTimeoutSec = 60,
|
|
120
|
+
candidateRetries = 2,
|
|
121
|
+
prices = {},
|
|
122
|
+
signal = null,
|
|
123
|
+
onProgress = null,
|
|
124
|
+
cwd = process.cwd(),
|
|
125
|
+
runId = '',
|
|
126
|
+
ctx = null,
|
|
127
|
+
}) {
|
|
128
|
+
if (typeof onProgress === 'function') {
|
|
129
|
+
onProgress(`🗣️ *Rebuttal Debate Round ${roundNumber}/${totalRounds}: Candidates cross-examining peer solutions in parallel...*\n`)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const roundCritiques = []
|
|
133
|
+
|
|
134
|
+
const debatePromises = candidates.map(async (cand) => {
|
|
135
|
+
const opponents = candidates
|
|
136
|
+
.filter((c) => c.index !== cand.index)
|
|
137
|
+
.map((c) => ({
|
|
138
|
+
index: c.index,
|
|
139
|
+
label: isBlindEvaluation ? `Candidate ${c.index}` : c.label,
|
|
140
|
+
role_persona: c.role_persona || c.slot?.role_persona,
|
|
141
|
+
text: c.text,
|
|
142
|
+
}))
|
|
143
|
+
|
|
144
|
+
const rebuttalPrompt = buildRebuttalPrompt({
|
|
145
|
+
userPrompt,
|
|
146
|
+
candidateText: cand.text,
|
|
147
|
+
opponentAnswers: opponents,
|
|
148
|
+
roundNumber,
|
|
149
|
+
totalRounds,
|
|
150
|
+
isBlind: isBlindEvaluation,
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
const slot = cand.slot || { provider: cand.provider, model: cand.model }
|
|
154
|
+
|
|
155
|
+
try {
|
|
156
|
+
const res = await callWithTransientRetry(
|
|
157
|
+
callLlm,
|
|
158
|
+
{
|
|
159
|
+
provider: slot.provider,
|
|
160
|
+
model: slot.model,
|
|
161
|
+
messages: [{ role: 'user', content: rebuttalPrompt }],
|
|
162
|
+
temperature: refTemp,
|
|
163
|
+
maxTokens,
|
|
164
|
+
timeoutMs: refTimeoutSec * 1000,
|
|
165
|
+
signal,
|
|
166
|
+
onProgress,
|
|
167
|
+
},
|
|
168
|
+
candidateRetries
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
const parsed = parseRebuttalResponse(res.text)
|
|
172
|
+
const costEst = estimateTokenCost({
|
|
173
|
+
provider: slot.provider,
|
|
174
|
+
model: slot.model,
|
|
175
|
+
inputTokens: res.usage?.inputTokens || 0,
|
|
176
|
+
outputTokens: res.usage?.outputTokens || 0,
|
|
177
|
+
cacheHitTokens: res.usage?.cacheHitTokens || 0,
|
|
178
|
+
prices,
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
const newFiles = extractFileBlocks(parsed.refinedText)
|
|
182
|
+
if (newFiles.length > 0) {
|
|
183
|
+
await writeCandidateWorkspace(cwd, cand.index, newFiles, { runId })
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const syntaxValidation = verifyFileSyntax(newFiles)
|
|
187
|
+
|
|
188
|
+
const critiqueRecord = {
|
|
189
|
+
candidateIndex: cand.index,
|
|
190
|
+
candidateLabel: isBlindEvaluation ? `Candidate ${cand.index}` : cand.label,
|
|
191
|
+
critique: parsed.critiqueText,
|
|
192
|
+
defense: parsed.defenseText,
|
|
193
|
+
}
|
|
194
|
+
roundCritiques.push(critiqueRecord)
|
|
195
|
+
|
|
196
|
+
return {
|
|
197
|
+
...cand,
|
|
198
|
+
text: parsed.refinedText,
|
|
199
|
+
files: newFiles.length > 0 ? newFiles : cand.files,
|
|
200
|
+
syntaxValidation: syntaxValidation.valid ? syntaxValidation : cand.syntaxValidation,
|
|
201
|
+
usage: {
|
|
202
|
+
inputTokens: (cand.usage?.inputTokens || 0) + (res.usage?.inputTokens || 0),
|
|
203
|
+
outputTokens: (cand.usage?.outputTokens || 0) + (res.usage?.outputTokens || 0),
|
|
204
|
+
totalTokens: (cand.usage?.totalTokens || 0) + (res.usage?.totalTokens || 0),
|
|
205
|
+
},
|
|
206
|
+
costUsd: Number(((cand.costUsd || 0) + (costEst?.costUsd || 0)).toFixed(5)),
|
|
207
|
+
lastDebateCritique: parsed.critiqueText,
|
|
208
|
+
lastDebateDefense: parsed.defenseText,
|
|
209
|
+
}
|
|
210
|
+
} catch (err) {
|
|
211
|
+
logger.warn(`[dsh-moa] Candidate ${cand.index} rebuttal debate round ${roundNumber} failed:`, err?.message || err)
|
|
212
|
+
return cand
|
|
213
|
+
}
|
|
214
|
+
})
|
|
215
|
+
|
|
216
|
+
const updatedCandidates = await Promise.all(debatePromises)
|
|
217
|
+
|
|
218
|
+
emitEvent(ctx, MOA_EVENTS.DEBATE_ROUND, {
|
|
219
|
+
runId,
|
|
220
|
+
roundNumber,
|
|
221
|
+
totalRounds,
|
|
222
|
+
critiquesCount: roundCritiques.length,
|
|
223
|
+
activeCandidates: updatedCandidates.length,
|
|
224
|
+
})
|
|
225
|
+
|
|
226
|
+
return {
|
|
227
|
+
round: roundNumber,
|
|
228
|
+
critiques: roundCritiques,
|
|
229
|
+
updatedCandidates,
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Formats accumulated debate protocol into a clean markdown directive for the Judge.
|
|
235
|
+
* @param {Array<object>} debateRounds
|
|
236
|
+
* @returns {string}
|
|
237
|
+
*/
|
|
238
|
+
export function formatDebateDirective(debateRounds = []) {
|
|
239
|
+
if (!Array.isArray(debateRounds) || debateRounds.length === 0) return ''
|
|
240
|
+
|
|
241
|
+
const sections = []
|
|
242
|
+
sections.push('=== ⚔️ MULTI-ROUND REBUTTAL DEBATE PROTOCOL ===')
|
|
243
|
+
sections.push('The candidate models engaged in peer cross-examination prior to arbitration.')
|
|
244
|
+
sections.push('CRITICAL JUDGE DIRECTIVE: Heavily weight valid vulnerabilities or bugs identified by peers.')
|
|
245
|
+
|
|
246
|
+
for (const r of debateRounds) {
|
|
247
|
+
sections.push(`\n--- Debate Round ${r.round} ---`)
|
|
248
|
+
for (const c of r.critiques || []) {
|
|
249
|
+
if (c.critique) {
|
|
250
|
+
sections.push(`[${c.candidateLabel}] Critiques of peers:\n${c.critique}`)
|
|
251
|
+
}
|
|
252
|
+
if (c.defense) {
|
|
253
|
+
sections.push(`[${c.candidateLabel}] Defense & Justification:\n${c.defense}`)
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
sections.push('\n==============================================')
|
|
259
|
+
return sections.join('\n')
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Orchestrates multi-round rebuttal debate phase (#298).
|
|
264
|
+
* @param {object} params
|
|
265
|
+
* @returns {Promise<{ ok: boolean, updatedCandidates: Array<object>, debateDirective: string, totalCritiques: number, rounds: Array<object> }>}
|
|
266
|
+
*/
|
|
267
|
+
export async function runRebuttalDebatePhase({
|
|
268
|
+
candidates = [],
|
|
269
|
+
userPrompt = '',
|
|
270
|
+
preset = {},
|
|
271
|
+
prices = {},
|
|
272
|
+
maxTokens = 4096,
|
|
273
|
+
isBlindEvaluation = true,
|
|
274
|
+
callLlm,
|
|
275
|
+
signal = null,
|
|
276
|
+
onProgress = null,
|
|
277
|
+
cwd = process.cwd(),
|
|
278
|
+
runId = '',
|
|
279
|
+
ctx = null,
|
|
280
|
+
}) {
|
|
281
|
+
const enabled = Boolean(preset?.rebuttal_debate_enabled || preset?.rebuttalDebateEnabled)
|
|
282
|
+
if (!enabled || candidates.length < 2) {
|
|
283
|
+
return {
|
|
284
|
+
ok: true,
|
|
285
|
+
updatedCandidates: candidates,
|
|
286
|
+
debateDirective: '',
|
|
287
|
+
totalCritiques: 0,
|
|
288
|
+
rounds: [],
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
const configuredRounds = preset?.rebuttal_rounds ?? preset?.rebuttalRounds ?? 1
|
|
293
|
+
const totalRounds = Math.min(3, Math.max(1, Number(configuredRounds) || 1))
|
|
294
|
+
|
|
295
|
+
let currentCandidates = [...candidates]
|
|
296
|
+
const debateRounds = []
|
|
297
|
+
|
|
298
|
+
for (let r = 1; r <= totalRounds; r++) {
|
|
299
|
+
if (signal?.aborted) {
|
|
300
|
+
break
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const roundResult = await executeDebateRound({
|
|
304
|
+
candidates: currentCandidates,
|
|
305
|
+
userPrompt,
|
|
306
|
+
roundNumber: r,
|
|
307
|
+
totalRounds,
|
|
308
|
+
isBlindEvaluation,
|
|
309
|
+
callLlm,
|
|
310
|
+
refTemp: preset?.reference_temperature ?? 0.5,
|
|
311
|
+
maxTokens,
|
|
312
|
+
refTimeoutSec: preset?.reference_timeout_sec ?? 60,
|
|
313
|
+
prices,
|
|
314
|
+
signal,
|
|
315
|
+
onProgress,
|
|
316
|
+
cwd,
|
|
317
|
+
runId,
|
|
318
|
+
ctx,
|
|
319
|
+
})
|
|
320
|
+
|
|
321
|
+
debateRounds.push(roundResult)
|
|
322
|
+
currentCandidates = roundResult.updatedCandidates
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
const totalCritiques = debateRounds.reduce((acc, r) => acc + (r.critiques?.length || 0), 0)
|
|
326
|
+
const debateDirective = formatDebateDirective(debateRounds)
|
|
327
|
+
|
|
328
|
+
return {
|
|
329
|
+
ok: true,
|
|
330
|
+
updatedCandidates: currentCandidates,
|
|
331
|
+
debateDirective,
|
|
332
|
+
totalCritiques,
|
|
333
|
+
rounds: debateRounds,
|
|
334
|
+
}
|
|
335
|
+
}
|
package/lib/pipeline/runner.js
CHANGED
|
@@ -25,6 +25,8 @@ import { runSecurityAuditGate } from './security-fuzzer.js'
|
|
|
25
25
|
import { runCoveVerificationGate } from './cove-verifier.js'
|
|
26
26
|
import { runProfilerGate } from './runtime-profiler.js'
|
|
27
27
|
import { runMutationGate } from './mutation-tester.js'
|
|
28
|
+
import { stageEmbeddingClustering } from './embedding-clustering.js'
|
|
29
|
+
import { runRebuttalDebatePhase } from './rebuttal-debate.js'
|
|
28
30
|
import { emitEvent, MOA_EVENTS } from '../events.js'
|
|
29
31
|
|
|
30
32
|
export async function runMoAPipeline({
|
|
@@ -119,7 +121,7 @@ export async function runMoAPipeline({
|
|
|
119
121
|
})
|
|
120
122
|
return candStage.failurePayload
|
|
121
123
|
}
|
|
122
|
-
|
|
124
|
+
let { referenceOutputs, successfulRefs, adversarialTddResult } = candStage
|
|
123
125
|
|
|
124
126
|
for (const ref of referenceOutputs) {
|
|
125
127
|
emitEvent(ctx, MOA_EVENTS.CANDIDATE_COMPLETE, {
|
|
@@ -202,6 +204,32 @@ export async function runMoAPipeline({
|
|
|
202
204
|
}
|
|
203
205
|
}
|
|
204
206
|
|
|
207
|
+
// Pre-Judge Stage 4.6: Blind Embedding Clustering for Zero-LLM Fast Consensus (#312)
|
|
208
|
+
if ((preset?.embedding_clustering_enabled || preset?.embeddingClusteringEnabled) && successfulRefs.length > 1) {
|
|
209
|
+
const embedRes = await stageEmbeddingClustering({
|
|
210
|
+
successfulRefs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
211
|
+
referenceOutputs,
|
|
212
|
+
preset,
|
|
213
|
+
userPrompt,
|
|
214
|
+
prices,
|
|
215
|
+
maxTokens: ctxStage.maxTokens,
|
|
216
|
+
cwd,
|
|
217
|
+
runId,
|
|
218
|
+
signal,
|
|
219
|
+
promoteOpts,
|
|
220
|
+
allowCandidateOverride: ctxStage.allowCandidateOverride,
|
|
221
|
+
isRefinement: ctxStage.isRefinement,
|
|
222
|
+
historyFilePath,
|
|
223
|
+
startTime,
|
|
224
|
+
collectedCtx: ctxStage.collectedCtx,
|
|
225
|
+
liveCanvas,
|
|
226
|
+
ctx,
|
|
227
|
+
})
|
|
228
|
+
if (embedRes.consensusReached) {
|
|
229
|
+
return embedRes.payload
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
205
233
|
if (signal?.aborted) {
|
|
206
234
|
await cleanMoaWorkspaces(cwd, { runId })
|
|
207
235
|
return { error: new Error('Turn aborted') }
|
|
@@ -228,6 +256,35 @@ export async function runMoAPipeline({
|
|
|
228
256
|
})
|
|
229
257
|
}
|
|
230
258
|
|
|
259
|
+
// Stage 5.2: Multi-Round Rebuttal Debate between Candidates (#298)
|
|
260
|
+
let debateDirective = null
|
|
261
|
+
if ((preset?.rebuttal_debate_enabled || preset?.rebuttalDebateEnabled) && successfulRefs.length > 1) {
|
|
262
|
+
const debateRes = await runRebuttalDebatePhase({
|
|
263
|
+
candidates: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
264
|
+
userPrompt,
|
|
265
|
+
preset,
|
|
266
|
+
prices,
|
|
267
|
+
maxTokens: ctxStage.maxTokens,
|
|
268
|
+
isBlindEvaluation: ctxStage.isBlindEvaluation,
|
|
269
|
+
callLlm,
|
|
270
|
+
signal,
|
|
271
|
+
onProgress,
|
|
272
|
+
cwd,
|
|
273
|
+
runId,
|
|
274
|
+
ctx,
|
|
275
|
+
})
|
|
276
|
+
if (debateRes.ok && debateRes.debateDirective) {
|
|
277
|
+
debateDirective = debateRes.debateDirective
|
|
278
|
+
if (debateRes.updatedCandidates?.length > 0) {
|
|
279
|
+
successfulRefs = debateRes.updatedCandidates
|
|
280
|
+
referenceOutputs = referenceOutputs.map((r) => {
|
|
281
|
+
const matching = successfulRefs.find((c) => c.index === r.index || c.label === r.label)
|
|
282
|
+
return matching ? { ...r, text: matching.text } : r
|
|
283
|
+
})
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
231
288
|
if (signal?.aborted) {
|
|
232
289
|
await cleanMoaWorkspaces(cwd, { runId })
|
|
233
290
|
return { error: new Error('Turn aborted') }
|
|
@@ -372,7 +429,7 @@ export async function runMoAPipeline({
|
|
|
372
429
|
preset: effectivePreset,
|
|
373
430
|
successfulRefs: candidatesForJudge,
|
|
374
431
|
referenceOutputs: sanitizedReferenceOutputs, primaryJudge: effectivePrimaryJudge,
|
|
375
|
-
userPrompt, judgeCriteria: [ctxStage.judgeCriteria, coveDirective, securityDirective, profilerDirective, mutationDirective, privacyDirective].filter(Boolean).join('\n\n'), prices, maxTokens: ctxStage.maxTokens,
|
|
432
|
+
userPrompt, judgeCriteria: [ctxStage.judgeCriteria, debateDirective, coveDirective, securityDirective, profilerDirective, mutationDirective, privacyDirective].filter(Boolean).join('\n\n'), prices, maxTokens: ctxStage.maxTokens,
|
|
376
433
|
callLlm, signal, onProgress, cwd, runId, isRefinement: ctxStage.isRefinement,
|
|
377
434
|
isFastMode: ctxStage.isFastMode, collectedCtx: ctxStage.collectedCtx, startTime,
|
|
378
435
|
})
|
|
@@ -430,7 +487,7 @@ export async function runMoAPipeline({
|
|
|
430
487
|
}
|
|
431
488
|
}
|
|
432
489
|
|
|
433
|
-
const fileScoringDirective = [rawFileDirective, visualResult?.directive, adversarialTddResult?.directive, syntaxDirective, coveDirective, securityDirective, profilerDirective, mutationDirective, privacyDirective, pruningDirective, ctxStage.stackAwareDirective, ctxStage.developerPreferencesDirective].filter(Boolean).join("\n\n") || null
|
|
490
|
+
const fileScoringDirective = [rawFileDirective, visualResult?.directive, adversarialTddResult?.directive, syntaxDirective, debateDirective, coveDirective, securityDirective, profilerDirective, mutationDirective, privacyDirective, pruningDirective, ctxStage.stackAwareDirective, ctxStage.developerPreferencesDirective].filter(Boolean).join("\n\n") || null
|
|
434
491
|
|
|
435
492
|
// Stage 7: Synthesis Phase
|
|
436
493
|
const synthRes = await stageSynthesis({
|