@goodandready/dsh-moa 0.2.75 → 0.2.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/README.ru.md +4 -0
- package/README.zh.md +4 -0
- package/lib/client.js +102 -2
- package/lib/events.js +4 -1
- package/lib/moa-schema.js +18 -1
- package/lib/pipeline/cove-verifier.js +401 -0
- package/lib/pipeline/mutation-tester.js +323 -0
- package/lib/pipeline/runner.js +72 -2
- package/lib/pipeline/runtime-profiler.js +270 -0
- package/lib/pipeline/security-fuzzer.js +290 -0
- package/package.json +1 -1
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DeepSeek Harness Mixture of Agents (MoA) — Mutation Testing Gate for Candidate Tests (#316)
|
|
3
|
+
*
|
|
4
|
+
* Injects deterministic AST-level fault mutations into candidate implementations and evaluates
|
|
5
|
+
* candidate test suites against mutants to calculate Mutation Score (killed vs survived mutants),
|
|
6
|
+
* eliminating false-positive and tautological tests before Judge arbitration.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import vm from 'node:vm'
|
|
10
|
+
import { MOA_EVENTS, emitEvent } from '../events.js'
|
|
11
|
+
import { extractExecutableCode } from './runtime-profiler.js'
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Mutation operator patterns.
|
|
15
|
+
*/
|
|
16
|
+
export const MUTATION_OPERATORS = [
|
|
17
|
+
{
|
|
18
|
+
name: 'invert_strict_equality',
|
|
19
|
+
regex: /===/g,
|
|
20
|
+
replacement: '!==',
|
|
21
|
+
description: 'Replaced strict equality (===) with inequality (!==)',
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
name: 'invert_relational_lt',
|
|
25
|
+
regex: /<(?!=)/g,
|
|
26
|
+
replacement: '>=',
|
|
27
|
+
description: 'Replaced less-than (<) with greater-or-equal (>=)',
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
name: 'invert_relational_gt',
|
|
31
|
+
regex: />(?!=)/g,
|
|
32
|
+
replacement: '<=',
|
|
33
|
+
description: 'Replaced greater-than (>) with less-or-equal (<=)',
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
name: 'invert_logical_and',
|
|
37
|
+
regex: /&&/g,
|
|
38
|
+
replacement: '||',
|
|
39
|
+
description: 'Replaced logical AND (&&) with logical OR (||)',
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
name: 'invert_arithmetic_plus',
|
|
43
|
+
regex: /(?<=\w\s*)\+(?=\s*\w)/g,
|
|
44
|
+
replacement: '-',
|
|
45
|
+
description: 'Replaced addition (+) with subtraction (-)',
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
name: 'invert_boolean_true',
|
|
49
|
+
regex: /\btrue\b/g,
|
|
50
|
+
replacement: 'false',
|
|
51
|
+
description: 'Replaced literal true with false',
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
name: 'mutate_return_zero',
|
|
55
|
+
regex: /return\s+0;/g,
|
|
56
|
+
replacement: 'return 1;',
|
|
57
|
+
description: 'Mutated return 0 to return 1',
|
|
58
|
+
},
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Generates an array of mutant variations from a single source code string.
|
|
63
|
+
*
|
|
64
|
+
* @param {string} sourceCode
|
|
65
|
+
* @param {object} [options]
|
|
66
|
+
* @returns {Array<object>} Up to maxMutants mutant objects
|
|
67
|
+
*/
|
|
68
|
+
export function generateMutants(sourceCode, options = {}) {
|
|
69
|
+
const maxMutants = Math.min(10, Math.max(1, options.maxMutants || 5))
|
|
70
|
+
const mutants = []
|
|
71
|
+
if (!sourceCode || typeof sourceCode !== 'string') return mutants
|
|
72
|
+
|
|
73
|
+
for (const op of MUTATION_OPERATORS) {
|
|
74
|
+
let match
|
|
75
|
+
const re = new RegExp(op.regex.source, op.regex.flags)
|
|
76
|
+
while ((match = re.exec(sourceCode)) !== null) {
|
|
77
|
+
const idx = match.index
|
|
78
|
+
const len = match[0].length
|
|
79
|
+
const mutantCode = sourceCode.slice(0, idx) + op.replacement + sourceCode.slice(idx + len)
|
|
80
|
+
|
|
81
|
+
mutants.push({
|
|
82
|
+
id: mutants.length + 1,
|
|
83
|
+
operator: op.name,
|
|
84
|
+
description: op.description,
|
|
85
|
+
originalSnippet: match[0],
|
|
86
|
+
replacementSnippet: op.replacement,
|
|
87
|
+
mutantCode,
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
if (mutants.length >= maxMutants) return mutants
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return mutants
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Runs a test script against an implementation in an isolated VM context.
|
|
99
|
+
*
|
|
100
|
+
* @param {string} codeToExecute
|
|
101
|
+
* @param {number} [timeoutMs=1000]
|
|
102
|
+
* @returns {{ ok: boolean, error: string | null }}
|
|
103
|
+
*/
|
|
104
|
+
function executeScript(codeToExecute, timeoutMs = 1000) {
|
|
105
|
+
const sandbox = {
|
|
106
|
+
console: { log: () => {}, warn: () => {}, error: () => {} },
|
|
107
|
+
assert: (cond, msg) => { if (!cond) throw new Error(msg || 'Assertion failed'); },
|
|
108
|
+
Math, Date, Array, Object, String, Number, Boolean, Set, Map, JSON,
|
|
109
|
+
}
|
|
110
|
+
sandbox.assert.equal = (a, b) => { if (a !== b) throw new Error(`Assertion failed: ${a} !== ${b}`); }
|
|
111
|
+
sandbox.assert.strictEqual = (a, b) => { if (a !== b) throw new Error(`Assertion failed: ${a} !== ${b}`); }
|
|
112
|
+
sandbox.assert.ok = (v) => { if (!v) throw new Error('Assertion failed: falsy value'); }
|
|
113
|
+
|
|
114
|
+
try {
|
|
115
|
+
const context = vm.createContext(sandbox)
|
|
116
|
+
const script = new vm.Script(codeToExecute)
|
|
117
|
+
script.runInContext(context, { timeout: timeoutMs })
|
|
118
|
+
return { ok: true, error: null }
|
|
119
|
+
} catch (err) {
|
|
120
|
+
return { ok: false, error: err.message }
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Evaluates a candidate's implementation against generated mutants.
|
|
126
|
+
*
|
|
127
|
+
* @param {string} rawText - Candidate text containing code or tests
|
|
128
|
+
* @param {object} [options]
|
|
129
|
+
* @returns {object} Mutation testing report
|
|
130
|
+
*/
|
|
131
|
+
export function auditCandidateMutations(rawText, options = {}) {
|
|
132
|
+
const { minMutationScore = 0.7, maxMutants = 5, timeoutMs = 1000 } = options
|
|
133
|
+
const code = extractExecutableCode(rawText)
|
|
134
|
+
|
|
135
|
+
if (!code || !code.trim()) {
|
|
136
|
+
return {
|
|
137
|
+
passed: true,
|
|
138
|
+
mutationScore: 100,
|
|
139
|
+
totalMutants: 0,
|
|
140
|
+
killedCount: 0,
|
|
141
|
+
survivedCount: 0,
|
|
142
|
+
mutants: [],
|
|
143
|
+
summary: 'Mutation Testing: No code to mutate.',
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// 1. Verify baseline executes cleanly
|
|
148
|
+
const baseline = executeScript(code, timeoutMs)
|
|
149
|
+
if (!baseline.ok) {
|
|
150
|
+
return {
|
|
151
|
+
passed: false,
|
|
152
|
+
mutationScore: 0,
|
|
153
|
+
totalMutants: 0,
|
|
154
|
+
killedCount: 0,
|
|
155
|
+
survivedCount: 0,
|
|
156
|
+
mutants: [],
|
|
157
|
+
error: `Baseline test execution failed: ${baseline.error}`,
|
|
158
|
+
summary: `Baseline execution failed (${baseline.error}).`,
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// 2. Generate mutations on the code
|
|
163
|
+
const mutants = generateMutants(code, { maxMutants })
|
|
164
|
+
if (mutants.length === 0) {
|
|
165
|
+
return {
|
|
166
|
+
passed: true,
|
|
167
|
+
mutationScore: 100,
|
|
168
|
+
totalMutants: 0,
|
|
169
|
+
killedCount: 0,
|
|
170
|
+
survivedCount: 0,
|
|
171
|
+
mutants: [],
|
|
172
|
+
summary: 'Mutation Testing: Code contains no mutable operators.',
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// 3. Test each mutant
|
|
177
|
+
let killedCount = 0
|
|
178
|
+
let survivedCount = 0
|
|
179
|
+
const evaluatedMutants = []
|
|
180
|
+
|
|
181
|
+
for (const m of mutants) {
|
|
182
|
+
const res = executeScript(m.mutantCode, timeoutMs)
|
|
183
|
+
if (!res.ok) {
|
|
184
|
+
// Mutant was killed (good! The test broke as expected)
|
|
185
|
+
killedCount++
|
|
186
|
+
evaluatedMutants.push({
|
|
187
|
+
...m,
|
|
188
|
+
status: 'KILLED',
|
|
189
|
+
reason: res.error,
|
|
190
|
+
})
|
|
191
|
+
} else {
|
|
192
|
+
// Mutant survived (bad! Code was broken but test passed anyway)
|
|
193
|
+
survivedCount++
|
|
194
|
+
evaluatedMutants.push({
|
|
195
|
+
...m,
|
|
196
|
+
status: 'SURVIVED',
|
|
197
|
+
reason: 'Test suite passed despite injected fault.',
|
|
198
|
+
})
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const mutationRatio = Number((killedCount / mutants.length).toFixed(2))
|
|
203
|
+
const mutationScore = Math.round(mutationRatio * 100)
|
|
204
|
+
const passed = mutationRatio >= minMutationScore
|
|
205
|
+
|
|
206
|
+
const summary = passed
|
|
207
|
+
? `🧬 Mutation Testing: PASSED (Score ${mutationScore}%). Killed ${killedCount}/${mutants.length} mutant(s).`
|
|
208
|
+
: `⚠️ Mutation Testing: WEAK TESTS (Score ${mutationScore}% < ${Math.round(minMutationScore * 100)}%). ${survivedCount} mutant(s) survived undetected.`
|
|
209
|
+
|
|
210
|
+
return {
|
|
211
|
+
passed,
|
|
212
|
+
mutationScore,
|
|
213
|
+
mutationRatio,
|
|
214
|
+
totalMutants: mutants.length,
|
|
215
|
+
killedCount,
|
|
216
|
+
survivedCount,
|
|
217
|
+
mutants: evaluatedMutants,
|
|
218
|
+
summary,
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Runs the Mutation Testing Gate across all candidate solutions.
|
|
224
|
+
*
|
|
225
|
+
* @param {Array<object>} candidates
|
|
226
|
+
* @param {object} [options]
|
|
227
|
+
* @returns {object} Mutation gate report
|
|
228
|
+
*/
|
|
229
|
+
export function runMutationGate(candidates, options = {}) {
|
|
230
|
+
const { ctx, runId, onProgress, minMutationScore = 0.70 } = options
|
|
231
|
+
const results = []
|
|
232
|
+
|
|
233
|
+
if (!Array.isArray(candidates) || candidates.length === 0) {
|
|
234
|
+
return {
|
|
235
|
+
results: [],
|
|
236
|
+
passedCount: 0,
|
|
237
|
+
mutationDirective: null,
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
for (let i = 0; i < candidates.length; i++) {
|
|
242
|
+
const cand = candidates[i]
|
|
243
|
+
const text = typeof cand === 'string' ? cand : cand.output || cand.text || cand.code || ''
|
|
244
|
+
const report = auditCandidateMutations(text, { minMutationScore, ...options })
|
|
245
|
+
|
|
246
|
+
if (typeof cand === 'object' && cand !== null) {
|
|
247
|
+
cand.mutationReport = report
|
|
248
|
+
cand.mutationScore = report.mutationScore
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
results.push({
|
|
252
|
+
candidateIndex: i,
|
|
253
|
+
model: cand?.model || `Candidate ${i + 1}`,
|
|
254
|
+
report,
|
|
255
|
+
})
|
|
256
|
+
|
|
257
|
+
if (ctx) {
|
|
258
|
+
emitEvent(ctx, MOA_EVENTS.MUTATION_COMPLETE, {
|
|
259
|
+
runId,
|
|
260
|
+
candidateIndex: i,
|
|
261
|
+
mutationScore: report.mutationScore,
|
|
262
|
+
killedCount: report.killedCount,
|
|
263
|
+
totalMutants: report.totalMutants,
|
|
264
|
+
passed: report.passed,
|
|
265
|
+
})
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const passedCount = results.filter((r) => r.report.passed).length
|
|
270
|
+
|
|
271
|
+
if (typeof onProgress === 'function' && results.length > 0) {
|
|
272
|
+
const summaryList = results.map((r) => `Ref ${r.candidateIndex + 1}: ${r.report.mutationScore}%`).join(', ')
|
|
273
|
+
onProgress(`🧬 *Mutation Testing Gate*: ${summaryList}\n`)
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const mutationDirective = formatMutationDirective(results)
|
|
277
|
+
|
|
278
|
+
return {
|
|
279
|
+
results,
|
|
280
|
+
passedCount,
|
|
281
|
+
mutationDirective,
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Formats mutation testing results into a prompt directive for the Judge.
|
|
287
|
+
*
|
|
288
|
+
* @param {Array<object>} results
|
|
289
|
+
* @returns {string|null}
|
|
290
|
+
*/
|
|
291
|
+
export function formatMutationDirective(results) {
|
|
292
|
+
if (!Array.isArray(results) || results.length === 0) return null
|
|
293
|
+
|
|
294
|
+
const lines = [
|
|
295
|
+
'### 🧬 RUNTIME MUTATION TESTING & FAULT INJECTION REPORT',
|
|
296
|
+
'Evaluation of test robustness against synthetic fault injection (killing code mutants):',
|
|
297
|
+
]
|
|
298
|
+
|
|
299
|
+
for (const r of results) {
|
|
300
|
+
const label = `Candidate ${r.candidateIndex + 1}`
|
|
301
|
+
const rep = r.report
|
|
302
|
+
if (rep.totalMutants === 0) {
|
|
303
|
+
lines.push(`- **${label}**: ℹ️ No mutable operators found in test suite.`)
|
|
304
|
+
} else if (rep.passed) {
|
|
305
|
+
lines.push(
|
|
306
|
+
`- **${label}**: ✅ **RESILIENT TESTS** (Score ${rep.mutationScore}%) — Killed ${rep.killedCount}/${rep.totalMutants} fault mutants.`
|
|
307
|
+
)
|
|
308
|
+
} else {
|
|
309
|
+
lines.push(
|
|
310
|
+
`- **${label}**: ⚠️ **WEAK/TAUTOLOGICAL TESTS** (Score ${rep.mutationScore}%) — ${rep.survivedCount}/${rep.totalMutants} mutant(s) survived! The tests pass even when logic is inverted.`
|
|
311
|
+
)
|
|
312
|
+
for (const m of (rep.mutants || []).filter((m) => m.status === 'SURVIVED')) {
|
|
313
|
+
lines.push(` * ${m.description} [Survived]`)
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
lines.push(
|
|
319
|
+
'\n**Judge Quality Mandate**: Prefer solutions whose test suites have a high Mutation Score. Solutions with surviving mutants have weak or tautological tests that do not genuinely verify behavior.'
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
return lines.join('\n')
|
|
323
|
+
}
|
package/lib/pipeline/runner.js
CHANGED
|
@@ -18,6 +18,10 @@ import { stageSynthesis } from './synthesis.js'
|
|
|
18
18
|
import { stagePromotion } from './promotion.js'
|
|
19
19
|
import { applySyntaxFilter, formatSyntaxDirective } from './syntax-filter.js'
|
|
20
20
|
import { stageEarlyConsensus } from './early-consensus.js'
|
|
21
|
+
import { runSecurityAuditGate } from './security-fuzzer.js'
|
|
22
|
+
import { runCoveVerificationGate } from './cove-verifier.js'
|
|
23
|
+
import { runProfilerGate } from './runtime-profiler.js'
|
|
24
|
+
import { runMutationGate } from './mutation-tester.js'
|
|
21
25
|
import { emitEvent, MOA_EVENTS } from '../events.js'
|
|
22
26
|
|
|
23
27
|
export async function runMoAPipeline({
|
|
@@ -199,6 +203,72 @@ export async function runMoAPipeline({
|
|
|
199
203
|
return { error: new Error('Turn aborted') }
|
|
200
204
|
}
|
|
201
205
|
|
|
206
|
+
// Pre-Stage 5.8: Chain-of-Verification (CoVe) Fact-Checking Gate (#314)
|
|
207
|
+
let coveDirective = null
|
|
208
|
+
let coveGateResult = null
|
|
209
|
+
if (preset?.cove_verification_enabled) {
|
|
210
|
+
coveGateResult = runCoveVerificationGate(
|
|
211
|
+
syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
212
|
+
ctxStage,
|
|
213
|
+
{
|
|
214
|
+
ctx,
|
|
215
|
+
runId,
|
|
216
|
+
onProgress,
|
|
217
|
+
maxQuestions: preset.cove_max_questions || 5,
|
|
218
|
+
}
|
|
219
|
+
)
|
|
220
|
+
coveDirective = coveGateResult.coveDirective
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Pre-Stage 5.9: Adversarial White-Hat Security Fuzzing Gate (#302)
|
|
224
|
+
let securityDirective = null
|
|
225
|
+
let securityGateResult = null
|
|
226
|
+
if (preset?.security_gate_enabled) {
|
|
227
|
+
securityGateResult = runSecurityAuditGate(
|
|
228
|
+
syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
229
|
+
{
|
|
230
|
+
ctx,
|
|
231
|
+
runId,
|
|
232
|
+
onProgress,
|
|
233
|
+
strictVeto: preset.security_gate_strict_veto !== false,
|
|
234
|
+
}
|
|
235
|
+
)
|
|
236
|
+
securityDirective = securityGateResult.securityDirective
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// Pre-Stage 5.10: Runtime CPU & Memory Profiler Gate (#315)
|
|
240
|
+
let profilerDirective = null
|
|
241
|
+
let profilerGateResult = null
|
|
242
|
+
if (preset?.profiler_gate_enabled) {
|
|
243
|
+
profilerGateResult = runProfilerGate(
|
|
244
|
+
syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
245
|
+
{
|
|
246
|
+
ctx,
|
|
247
|
+
runId,
|
|
248
|
+
onProgress,
|
|
249
|
+
iterations: preset.profiler_iterations || 100,
|
|
250
|
+
timeoutMs: preset.profiler_timeout_ms || 2000,
|
|
251
|
+
}
|
|
252
|
+
)
|
|
253
|
+
profilerDirective = profilerGateResult.profilerDirective
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// Pre-Stage 5.11: Mutation Testing & Fault Injection Gate (#316)
|
|
257
|
+
let mutationDirective = null
|
|
258
|
+
let mutationGateResult = null
|
|
259
|
+
if (preset?.mutation_testing_enabled) {
|
|
260
|
+
mutationGateResult = runMutationGate(
|
|
261
|
+
syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
262
|
+
{
|
|
263
|
+
ctx,
|
|
264
|
+
runId,
|
|
265
|
+
onProgress,
|
|
266
|
+
minMutationScore: preset.min_mutation_score ?? 0.70,
|
|
267
|
+
}
|
|
268
|
+
)
|
|
269
|
+
mutationDirective = mutationGateResult.mutationDirective
|
|
270
|
+
}
|
|
271
|
+
|
|
202
272
|
// Pre-Stage 6: Dynamic Consensus Threshold & Multi-Judge Escalation (#300)
|
|
203
273
|
const escalationResult = evaluateJudgeEscalation({
|
|
204
274
|
candidateOutputs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
@@ -249,7 +319,7 @@ export async function runMoAPipeline({
|
|
|
249
319
|
preset: effectivePreset,
|
|
250
320
|
successfulRefs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
251
321
|
referenceOutputs, primaryJudge: effectivePrimaryJudge,
|
|
252
|
-
userPrompt, judgeCriteria: ctxStage.judgeCriteria, prices, maxTokens: ctxStage.maxTokens,
|
|
322
|
+
userPrompt, judgeCriteria: [ctxStage.judgeCriteria, coveDirective, securityDirective, profilerDirective, mutationDirective].filter(Boolean).join('\n\n'), prices, maxTokens: ctxStage.maxTokens,
|
|
253
323
|
callLlm, signal, onProgress, cwd, runId, isRefinement: ctxStage.isRefinement,
|
|
254
324
|
isFastMode: ctxStage.isFastMode, collectedCtx: ctxStage.collectedCtx, startTime,
|
|
255
325
|
})
|
|
@@ -307,7 +377,7 @@ export async function runMoAPipeline({
|
|
|
307
377
|
}
|
|
308
378
|
}
|
|
309
379
|
|
|
310
|
-
const fileScoringDirective = [rawFileDirective, visualResult?.directive, adversarialTddResult?.directive, syntaxDirective, pruningDirective, ctxStage.stackAwareDirective, ctxStage.developerPreferencesDirective].filter(Boolean).join("\n\n") || null
|
|
380
|
+
const fileScoringDirective = [rawFileDirective, visualResult?.directive, adversarialTddResult?.directive, syntaxDirective, coveDirective, securityDirective, profilerDirective, mutationDirective, pruningDirective, ctxStage.stackAwareDirective, ctxStage.developerPreferencesDirective].filter(Boolean).join("\n\n") || null
|
|
311
381
|
|
|
312
382
|
// Stage 7: Synthesis Phase
|
|
313
383
|
const synthRes = await stageSynthesis({
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DeepSeek Harness Mixture of Agents (MoA) — Runtime CPU & Memory Profiler Gate (#315)
|
|
3
|
+
*
|
|
4
|
+
* Micro-benchmarks candidate code execution duration, ops/sec throughput, and heap memory usage
|
|
5
|
+
* in an isolated sandbox with hard timeout protection to guide Judge selection towards
|
|
6
|
+
* algorithmically optimal solutions.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import vm from 'node:vm'
|
|
10
|
+
import { MOA_EVENTS, emitEvent } from '../events.js'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Extracts executable JavaScript code from markdown fences or raw code.
|
|
14
|
+
*
|
|
15
|
+
* @param {string} text
|
|
16
|
+
* @returns {string}
|
|
17
|
+
*/
|
|
18
|
+
export function extractExecutableCode(text) {
|
|
19
|
+
if (!text || typeof text !== 'string') return ''
|
|
20
|
+
const jsFenceMatch = text.match(/```(?:javascript|js|node)\s*\n([\s\S]*?)\n```/i)
|
|
21
|
+
if (jsFenceMatch && jsFenceMatch[1]) {
|
|
22
|
+
return jsFenceMatch[1]
|
|
23
|
+
}
|
|
24
|
+
const anyFenceMatch = text.match(/```\w*\s*\n([\s\S]*?)\n```/)
|
|
25
|
+
if (anyFenceMatch && anyFenceMatch[1]) {
|
|
26
|
+
return anyFenceMatch[1]
|
|
27
|
+
}
|
|
28
|
+
return text
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Profiles code execution in an isolated VM sandbox.
|
|
33
|
+
*
|
|
34
|
+
* @param {string} rawCode - Candidate source code or markdown
|
|
35
|
+
* @param {object} [options]
|
|
36
|
+
* @returns {object} Profiler report
|
|
37
|
+
*/
|
|
38
|
+
export function profileCodeExecution(rawCode, options = {}) {
|
|
39
|
+
const {
|
|
40
|
+
iterations = 100,
|
|
41
|
+
timeoutMs = 2000,
|
|
42
|
+
} = options
|
|
43
|
+
|
|
44
|
+
const code = extractExecutableCode(rawCode)
|
|
45
|
+
if (!code || !code.trim()) {
|
|
46
|
+
return {
|
|
47
|
+
passed: false,
|
|
48
|
+
durationMs: 0,
|
|
49
|
+
heapBytesUsed: 0,
|
|
50
|
+
opsPerSec: 0,
|
|
51
|
+
iterations: 0,
|
|
52
|
+
error: 'Empty or unexecutable code provided for profiling.',
|
|
53
|
+
summary: 'Profiler skipped: no executable code.',
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Initial sandbox built-ins to ignore when scanning for user functions
|
|
58
|
+
const initialKeys = new Set([
|
|
59
|
+
'console', 'Math', 'Date', 'Array', 'Object', 'String', 'Number',
|
|
60
|
+
'Boolean', 'Set', 'Map', 'JSON', 'parseInt', 'parseFloat',
|
|
61
|
+
'globalThis', 'global', '__mainFn', '__benchFn', '__benchArg'
|
|
62
|
+
])
|
|
63
|
+
|
|
64
|
+
const sandbox = {
|
|
65
|
+
console: { log: () => {}, warn: () => {}, error: () => {} },
|
|
66
|
+
Math,
|
|
67
|
+
Date,
|
|
68
|
+
Array,
|
|
69
|
+
Object,
|
|
70
|
+
String,
|
|
71
|
+
Number,
|
|
72
|
+
Boolean,
|
|
73
|
+
Set,
|
|
74
|
+
Map,
|
|
75
|
+
JSON,
|
|
76
|
+
parseInt,
|
|
77
|
+
parseFloat,
|
|
78
|
+
}
|
|
79
|
+
sandbox.globalThis = sandbox
|
|
80
|
+
sandbox.global = sandbox
|
|
81
|
+
|
|
82
|
+
// Convert export declarations to var / global assignments
|
|
83
|
+
const cleanCode = code
|
|
84
|
+
.replace(/^\s*export\s+default\s+/gm, 'globalThis.__mainFn = ')
|
|
85
|
+
.replace(/^\s*export\s+(?:async\s+)?function\s+([a-zA-Z0-9_]+)/gm, 'function $1')
|
|
86
|
+
.replace(/^\s*export\s+const\s+([a-zA-Z0-9_]+)/gm, 'var $1')
|
|
87
|
+
.replace(/^\s*import\s+.*?;?\s*$/gm, '// [import stripped for benchmark]')
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
const context = vm.createContext(sandbox)
|
|
91
|
+
// Run candidate code once to evaluate top-level declarations
|
|
92
|
+
const setupScript = new vm.Script(cleanCode)
|
|
93
|
+
setupScript.runInContext(context, { timeout: Math.min(1000, timeoutMs) })
|
|
94
|
+
|
|
95
|
+
// Find entrypoint function: either __mainFn or any user-defined function on sandbox
|
|
96
|
+
let targetFn = sandbox.__mainFn
|
|
97
|
+
if (typeof targetFn !== 'function') {
|
|
98
|
+
const userKeys = Object.keys(sandbox).filter((k) => !initialKeys.has(k) && typeof sandbox[k] === 'function')
|
|
99
|
+
if (userKeys.length > 0) {
|
|
100
|
+
targetFn = sandbox[userKeys[userKeys.length - 1]]
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
sandbox.__benchFn = targetFn || null
|
|
105
|
+
sandbox.__benchArg = 10
|
|
106
|
+
|
|
107
|
+
let benchScript
|
|
108
|
+
if (typeof sandbox.__benchFn === 'function') {
|
|
109
|
+
benchScript = new vm.Script(`
|
|
110
|
+
try {
|
|
111
|
+
__benchFn();
|
|
112
|
+
} catch (_) {
|
|
113
|
+
try { __benchFn(__benchArg); } catch (__) {}
|
|
114
|
+
}
|
|
115
|
+
`)
|
|
116
|
+
} else {
|
|
117
|
+
benchScript = new vm.Script(`(() => { ${cleanCode} })()`)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Warm-up run
|
|
121
|
+
benchScript.runInContext(context, { timeout: Math.min(1000, timeoutMs) })
|
|
122
|
+
|
|
123
|
+
// Measure memory and time
|
|
124
|
+
const startMem = process.memoryUsage().heapUsed
|
|
125
|
+
const startHr = process.hrtime.bigint()
|
|
126
|
+
|
|
127
|
+
for (let i = 0; i < iterations; i++) {
|
|
128
|
+
benchScript.runInContext(context, { timeout: Math.min(500, timeoutMs) })
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const endHr = process.hrtime.bigint()
|
|
132
|
+
const endMem = process.memoryUsage().heapUsed
|
|
133
|
+
|
|
134
|
+
const totalNs = Number(endHr - startHr)
|
|
135
|
+
const durationMs = Number((totalNs / 1_000_000).toFixed(3))
|
|
136
|
+
const avgLatencyMs = Number((durationMs / iterations).toFixed(4))
|
|
137
|
+
const heapBytesUsed = Math.max(0, endMem - startMem)
|
|
138
|
+
const opsPerSec = durationMs > 0 ? Math.round((iterations / (durationMs / 1000))) : 0
|
|
139
|
+
|
|
140
|
+
return {
|
|
141
|
+
passed: true,
|
|
142
|
+
durationMs,
|
|
143
|
+
avgLatencyMs,
|
|
144
|
+
heapBytesUsed,
|
|
145
|
+
opsPerSec,
|
|
146
|
+
iterations,
|
|
147
|
+
error: null,
|
|
148
|
+
summary: `⚡ Profiler: ${iterations} iter in ${durationMs}ms (${opsPerSec.toLocaleString()} ops/sec, ~${Math.round(heapBytesUsed / 1024)}KB heap).`,
|
|
149
|
+
}
|
|
150
|
+
} catch (err) {
|
|
151
|
+
const isTimeout = err.code === 'ERR_SCRIPT_EXECUTION_TIMEOUT' || err.message?.includes('timed out')
|
|
152
|
+
return {
|
|
153
|
+
passed: false,
|
|
154
|
+
durationMs: isTimeout ? timeoutMs : 0,
|
|
155
|
+
avgLatencyMs: 0,
|
|
156
|
+
heapBytesUsed: 0,
|
|
157
|
+
opsPerSec: 0,
|
|
158
|
+
iterations: 0,
|
|
159
|
+
error: isTimeout ? `Execution timed out after ${timeoutMs}ms (possible infinite loop or catastrophic complexity)` : err.message,
|
|
160
|
+
summary: isTimeout ? `🚨 Profiler: TIMEOUT (${timeoutMs}ms exceeded)` : `⚠️ Profiler Error: ${err.message}`,
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Runs runtime performance profiling across all candidate solutions.
|
|
167
|
+
*
|
|
168
|
+
* @param {Array<object>} candidates
|
|
169
|
+
* @param {object} [options]
|
|
170
|
+
* @returns {object} Profiler gate result
|
|
171
|
+
*/
|
|
172
|
+
export function runProfilerGate(candidates, options = {}) {
|
|
173
|
+
const {
|
|
174
|
+
ctx,
|
|
175
|
+
runId,
|
|
176
|
+
onProgress,
|
|
177
|
+
iterations = 100,
|
|
178
|
+
timeoutMs = 2000,
|
|
179
|
+
} = options
|
|
180
|
+
|
|
181
|
+
const results = []
|
|
182
|
+
if (!Array.isArray(candidates) || candidates.length === 0) {
|
|
183
|
+
return {
|
|
184
|
+
results: [],
|
|
185
|
+
fastestCandidate: null,
|
|
186
|
+
profilerDirective: null,
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
for (let i = 0; i < candidates.length; i++) {
|
|
191
|
+
const cand = candidates[i]
|
|
192
|
+
const text = typeof cand === 'string' ? cand : cand.output || cand.text || cand.code || ''
|
|
193
|
+
const report = profileCodeExecution(text, { iterations, timeoutMs })
|
|
194
|
+
|
|
195
|
+
if (typeof cand === 'object' && cand !== null) {
|
|
196
|
+
cand.profilerReport = report
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
results.push({
|
|
200
|
+
candidateIndex: i,
|
|
201
|
+
model: cand?.model || `Candidate ${i + 1}`,
|
|
202
|
+
report,
|
|
203
|
+
})
|
|
204
|
+
|
|
205
|
+
if (ctx) {
|
|
206
|
+
emitEvent(ctx, MOA_EVENTS.PROFILER_COMPLETE, {
|
|
207
|
+
runId,
|
|
208
|
+
candidateIndex: i,
|
|
209
|
+
durationMs: report.durationMs,
|
|
210
|
+
heapBytesUsed: report.heapBytesUsed,
|
|
211
|
+
opsPerSec: report.opsPerSec,
|
|
212
|
+
passed: report.passed,
|
|
213
|
+
error: report.error,
|
|
214
|
+
})
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Sort passing candidates by throughput (opsPerSec) descending
|
|
219
|
+
const valid = results.filter((r) => r.report.passed).sort((a, b) => b.report.opsPerSec - a.report.opsPerSec)
|
|
220
|
+
const fastestCandidate = valid[0] || null
|
|
221
|
+
|
|
222
|
+
if (typeof onProgress === 'function' && results.length > 0) {
|
|
223
|
+
const summaryList = results.map((r) => `Ref ${r.candidateIndex + 1}: ${r.report.passed ? `${r.report.durationMs}ms` : 'TIMEOUT'}`).join(', ')
|
|
224
|
+
onProgress(`⚡ *Runtime Profiler Gate*: ${summaryList}\n`)
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const profilerDirective = formatProfilerDirective(results)
|
|
228
|
+
|
|
229
|
+
return {
|
|
230
|
+
results,
|
|
231
|
+
fastestCandidate,
|
|
232
|
+
profilerDirective,
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Formats profiler benchmarks into a clear prompt directive for the Judge.
|
|
238
|
+
*
|
|
239
|
+
* @param {Array<object>} results
|
|
240
|
+
* @returns {string|null}
|
|
241
|
+
*/
|
|
242
|
+
export function formatProfilerDirective(results) {
|
|
243
|
+
if (!Array.isArray(results) || results.length === 0) return null
|
|
244
|
+
|
|
245
|
+
const lines = [
|
|
246
|
+
'### ⚡ RUNTIME PERFORMANCE & CPU/MEMORY PROFILER REPORT',
|
|
247
|
+
'Objective micro-benchmarks across candidates (100 iterations in isolated sandbox):',
|
|
248
|
+
]
|
|
249
|
+
|
|
250
|
+
for (const r of results) {
|
|
251
|
+
const label = `Candidate ${r.candidateIndex + 1}`
|
|
252
|
+
const rep = r.report
|
|
253
|
+
if (rep.passed) {
|
|
254
|
+
const kb = Math.round(rep.heapBytesUsed / 1024)
|
|
255
|
+
lines.push(
|
|
256
|
+
`- **${label}**: ⏱️ **${rep.durationMs}ms** total (~${rep.avgLatencyMs}ms/op) | **${rep.opsPerSec.toLocaleString()} ops/sec** | Heap: ~${kb} KB`
|
|
257
|
+
)
|
|
258
|
+
} else {
|
|
259
|
+
lines.push(
|
|
260
|
+
`- **${label}**: ❌ **PERFORMANCE FAILURE**: ${rep.error}`
|
|
261
|
+
)
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
lines.push(
|
|
266
|
+
'\n**Judge Performance Guideline**: Penalize solutions with excessive latency or timeout failures. Prefer implementations demonstrating clean algorithmic scalability and minimal memory footprint.'
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
return lines.join('\n')
|
|
270
|
+
}
|