@goodandready/dsh-moa 0.2.75 → 0.2.77

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,323 @@
1
+ /**
2
+ * DeepSeek Harness Mixture of Agents (MoA) — Mutation Testing Gate for Candidate Tests (#316)
3
+ *
4
+ * Injects deterministic AST-level fault mutations into candidate implementations and evaluates
5
+ * candidate test suites against mutants to calculate Mutation Score (killed vs survived mutants),
6
+ * eliminating false-positive and tautological tests before Judge arbitration.
7
+ */
8
+
9
+ import vm from 'node:vm'
10
+ import { MOA_EVENTS, emitEvent } from '../events.js'
11
+ import { extractExecutableCode } from './runtime-profiler.js'
12
+
13
+ /**
14
+ * Mutation operator patterns.
15
+ */
16
+ export const MUTATION_OPERATORS = [
17
+ {
18
+ name: 'invert_strict_equality',
19
+ regex: /===/g,
20
+ replacement: '!==',
21
+ description: 'Replaced strict equality (===) with inequality (!==)',
22
+ },
23
+ {
24
+ name: 'invert_relational_lt',
25
+ regex: /<(?!=)/g,
26
+ replacement: '>=',
27
+ description: 'Replaced less-than (<) with greater-or-equal (>=)',
28
+ },
29
+ {
30
+ name: 'invert_relational_gt',
31
+ regex: />(?!=)/g,
32
+ replacement: '<=',
33
+ description: 'Replaced greater-than (>) with less-or-equal (<=)',
34
+ },
35
+ {
36
+ name: 'invert_logical_and',
37
+ regex: /&&/g,
38
+ replacement: '||',
39
+ description: 'Replaced logical AND (&&) with logical OR (||)',
40
+ },
41
+ {
42
+ name: 'invert_arithmetic_plus',
43
+ regex: /(?<=\w\s*)\+(?=\s*\w)/g,
44
+ replacement: '-',
45
+ description: 'Replaced addition (+) with subtraction (-)',
46
+ },
47
+ {
48
+ name: 'invert_boolean_true',
49
+ regex: /\btrue\b/g,
50
+ replacement: 'false',
51
+ description: 'Replaced literal true with false',
52
+ },
53
+ {
54
+ name: 'mutate_return_zero',
55
+ regex: /return\s+0;/g,
56
+ replacement: 'return 1;',
57
+ description: 'Mutated return 0 to return 1',
58
+ },
59
+ ]
60
+
61
+ /**
62
+ * Generates an array of mutant variations from a single source code string.
63
+ *
64
+ * @param {string} sourceCode
65
+ * @param {object} [options]
66
+ * @returns {Array<object>} Up to maxMutants mutant objects
67
+ */
68
+ export function generateMutants(sourceCode, options = {}) {
69
+ const maxMutants = Math.min(10, Math.max(1, options.maxMutants || 5))
70
+ const mutants = []
71
+ if (!sourceCode || typeof sourceCode !== 'string') return mutants
72
+
73
+ for (const op of MUTATION_OPERATORS) {
74
+ let match
75
+ const re = new RegExp(op.regex.source, op.regex.flags)
76
+ while ((match = re.exec(sourceCode)) !== null) {
77
+ const idx = match.index
78
+ const len = match[0].length
79
+ const mutantCode = sourceCode.slice(0, idx) + op.replacement + sourceCode.slice(idx + len)
80
+
81
+ mutants.push({
82
+ id: mutants.length + 1,
83
+ operator: op.name,
84
+ description: op.description,
85
+ originalSnippet: match[0],
86
+ replacementSnippet: op.replacement,
87
+ mutantCode,
88
+ })
89
+
90
+ if (mutants.length >= maxMutants) return mutants
91
+ }
92
+ }
93
+
94
+ return mutants
95
+ }
96
+
97
+ /**
98
+ * Runs a test script against an implementation in an isolated VM context.
99
+ *
100
+ * @param {string} codeToExecute
101
+ * @param {number} [timeoutMs=1000]
102
+ * @returns {{ ok: boolean, error: string | null }}
103
+ */
104
+ function executeScript(codeToExecute, timeoutMs = 1000) {
105
+ const sandbox = {
106
+ console: { log: () => {}, warn: () => {}, error: () => {} },
107
+ assert: (cond, msg) => { if (!cond) throw new Error(msg || 'Assertion failed'); },
108
+ Math, Date, Array, Object, String, Number, Boolean, Set, Map, JSON,
109
+ }
110
+ sandbox.assert.equal = (a, b) => { if (a !== b) throw new Error(`Assertion failed: ${a} !== ${b}`); }
111
+ sandbox.assert.strictEqual = (a, b) => { if (a !== b) throw new Error(`Assertion failed: ${a} !== ${b}`); }
112
+ sandbox.assert.ok = (v) => { if (!v) throw new Error('Assertion failed: falsy value'); }
113
+
114
+ try {
115
+ const context = vm.createContext(sandbox)
116
+ const script = new vm.Script(codeToExecute)
117
+ script.runInContext(context, { timeout: timeoutMs })
118
+ return { ok: true, error: null }
119
+ } catch (err) {
120
+ return { ok: false, error: err.message }
121
+ }
122
+ }
123
+
124
+ /**
125
+ * Evaluates a candidate's implementation against generated mutants.
126
+ *
127
+ * @param {string} rawText - Candidate text containing code or tests
128
+ * @param {object} [options]
129
+ * @returns {object} Mutation testing report
130
+ */
131
+ export function auditCandidateMutations(rawText, options = {}) {
132
+ const { minMutationScore = 0.7, maxMutants = 5, timeoutMs = 1000 } = options
133
+ const code = extractExecutableCode(rawText)
134
+
135
+ if (!code || !code.trim()) {
136
+ return {
137
+ passed: true,
138
+ mutationScore: 100,
139
+ totalMutants: 0,
140
+ killedCount: 0,
141
+ survivedCount: 0,
142
+ mutants: [],
143
+ summary: 'Mutation Testing: No code to mutate.',
144
+ }
145
+ }
146
+
147
+ // 1. Verify baseline executes cleanly
148
+ const baseline = executeScript(code, timeoutMs)
149
+ if (!baseline.ok) {
150
+ return {
151
+ passed: false,
152
+ mutationScore: 0,
153
+ totalMutants: 0,
154
+ killedCount: 0,
155
+ survivedCount: 0,
156
+ mutants: [],
157
+ error: `Baseline test execution failed: ${baseline.error}`,
158
+ summary: `Baseline execution failed (${baseline.error}).`,
159
+ }
160
+ }
161
+
162
+ // 2. Generate mutations on the code
163
+ const mutants = generateMutants(code, { maxMutants })
164
+ if (mutants.length === 0) {
165
+ return {
166
+ passed: true,
167
+ mutationScore: 100,
168
+ totalMutants: 0,
169
+ killedCount: 0,
170
+ survivedCount: 0,
171
+ mutants: [],
172
+ summary: 'Mutation Testing: Code contains no mutable operators.',
173
+ }
174
+ }
175
+
176
+ // 3. Test each mutant
177
+ let killedCount = 0
178
+ let survivedCount = 0
179
+ const evaluatedMutants = []
180
+
181
+ for (const m of mutants) {
182
+ const res = executeScript(m.mutantCode, timeoutMs)
183
+ if (!res.ok) {
184
+ // Mutant was killed (good! The test broke as expected)
185
+ killedCount++
186
+ evaluatedMutants.push({
187
+ ...m,
188
+ status: 'KILLED',
189
+ reason: res.error,
190
+ })
191
+ } else {
192
+ // Mutant survived (bad! Code was broken but test passed anyway)
193
+ survivedCount++
194
+ evaluatedMutants.push({
195
+ ...m,
196
+ status: 'SURVIVED',
197
+ reason: 'Test suite passed despite injected fault.',
198
+ })
199
+ }
200
+ }
201
+
202
+ const mutationRatio = Number((killedCount / mutants.length).toFixed(2))
203
+ const mutationScore = Math.round(mutationRatio * 100)
204
+ const passed = mutationRatio >= minMutationScore
205
+
206
+ const summary = passed
207
+ ? `🧬 Mutation Testing: PASSED (Score ${mutationScore}%). Killed ${killedCount}/${mutants.length} mutant(s).`
208
+ : `⚠️ Mutation Testing: WEAK TESTS (Score ${mutationScore}% < ${Math.round(minMutationScore * 100)}%). ${survivedCount} mutant(s) survived undetected.`
209
+
210
+ return {
211
+ passed,
212
+ mutationScore,
213
+ mutationRatio,
214
+ totalMutants: mutants.length,
215
+ killedCount,
216
+ survivedCount,
217
+ mutants: evaluatedMutants,
218
+ summary,
219
+ }
220
+ }
221
+
222
+ /**
223
+ * Runs the Mutation Testing Gate across all candidate solutions.
224
+ *
225
+ * @param {Array<object>} candidates
226
+ * @param {object} [options]
227
+ * @returns {object} Mutation gate report
228
+ */
229
+ export function runMutationGate(candidates, options = {}) {
230
+ const { ctx, runId, onProgress, minMutationScore = 0.70 } = options
231
+ const results = []
232
+
233
+ if (!Array.isArray(candidates) || candidates.length === 0) {
234
+ return {
235
+ results: [],
236
+ passedCount: 0,
237
+ mutationDirective: null,
238
+ }
239
+ }
240
+
241
+ for (let i = 0; i < candidates.length; i++) {
242
+ const cand = candidates[i]
243
+ const text = typeof cand === 'string' ? cand : cand.output || cand.text || cand.code || ''
244
+ const report = auditCandidateMutations(text, { minMutationScore, ...options })
245
+
246
+ if (typeof cand === 'object' && cand !== null) {
247
+ cand.mutationReport = report
248
+ cand.mutationScore = report.mutationScore
249
+ }
250
+
251
+ results.push({
252
+ candidateIndex: i,
253
+ model: cand?.model || `Candidate ${i + 1}`,
254
+ report,
255
+ })
256
+
257
+ if (ctx) {
258
+ emitEvent(ctx, MOA_EVENTS.MUTATION_COMPLETE, {
259
+ runId,
260
+ candidateIndex: i,
261
+ mutationScore: report.mutationScore,
262
+ killedCount: report.killedCount,
263
+ totalMutants: report.totalMutants,
264
+ passed: report.passed,
265
+ })
266
+ }
267
+ }
268
+
269
+ const passedCount = results.filter((r) => r.report.passed).length
270
+
271
+ if (typeof onProgress === 'function' && results.length > 0) {
272
+ const summaryList = results.map((r) => `Ref ${r.candidateIndex + 1}: ${r.report.mutationScore}%`).join(', ')
273
+ onProgress(`🧬 *Mutation Testing Gate*: ${summaryList}\n`)
274
+ }
275
+
276
+ const mutationDirective = formatMutationDirective(results)
277
+
278
+ return {
279
+ results,
280
+ passedCount,
281
+ mutationDirective,
282
+ }
283
+ }
284
+
285
+ /**
286
+ * Formats mutation testing results into a prompt directive for the Judge.
287
+ *
288
+ * @param {Array<object>} results
289
+ * @returns {string|null}
290
+ */
291
+ export function formatMutationDirective(results) {
292
+ if (!Array.isArray(results) || results.length === 0) return null
293
+
294
+ const lines = [
295
+ '### 🧬 RUNTIME MUTATION TESTING & FAULT INJECTION REPORT',
296
+ 'Evaluation of test robustness against synthetic fault injection (killing code mutants):',
297
+ ]
298
+
299
+ for (const r of results) {
300
+ const label = `Candidate ${r.candidateIndex + 1}`
301
+ const rep = r.report
302
+ if (rep.totalMutants === 0) {
303
+ lines.push(`- **${label}**: ℹ️ No mutable operators found in test suite.`)
304
+ } else if (rep.passed) {
305
+ lines.push(
306
+ `- **${label}**: ✅ **RESILIENT TESTS** (Score ${rep.mutationScore}%) — Killed ${rep.killedCount}/${rep.totalMutants} fault mutants.`
307
+ )
308
+ } else {
309
+ lines.push(
310
+ `- **${label}**: ⚠️ **WEAK/TAUTOLOGICAL TESTS** (Score ${rep.mutationScore}%) — ${rep.survivedCount}/${rep.totalMutants} mutant(s) survived! The tests pass even when logic is inverted.`
311
+ )
312
+ for (const m of (rep.mutants || []).filter((m) => m.status === 'SURVIVED')) {
313
+ lines.push(` * ${m.description} [Survived]`)
314
+ }
315
+ }
316
+ }
317
+
318
+ lines.push(
319
+ '\n**Judge Quality Mandate**: Prefer solutions whose test suites have a high Mutation Score. Solutions with surviving mutants have weak or tautological tests that do not genuinely verify behavior.'
320
+ )
321
+
322
+ return lines.join('\n')
323
+ }
@@ -18,6 +18,10 @@ import { stageSynthesis } from './synthesis.js'
18
18
  import { stagePromotion } from './promotion.js'
19
19
  import { applySyntaxFilter, formatSyntaxDirective } from './syntax-filter.js'
20
20
  import { stageEarlyConsensus } from './early-consensus.js'
21
+ import { runSecurityAuditGate } from './security-fuzzer.js'
22
+ import { runCoveVerificationGate } from './cove-verifier.js'
23
+ import { runProfilerGate } from './runtime-profiler.js'
24
+ import { runMutationGate } from './mutation-tester.js'
21
25
  import { emitEvent, MOA_EVENTS } from '../events.js'
22
26
 
23
27
  export async function runMoAPipeline({
@@ -199,6 +203,72 @@ export async function runMoAPipeline({
199
203
  return { error: new Error('Turn aborted') }
200
204
  }
201
205
 
206
+ // Pre-Stage 5.8: Chain-of-Verification (CoVe) Fact-Checking Gate (#314)
207
+ let coveDirective = null
208
+ let coveGateResult = null
209
+ if (preset?.cove_verification_enabled) {
210
+ coveGateResult = runCoveVerificationGate(
211
+ syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
212
+ ctxStage,
213
+ {
214
+ ctx,
215
+ runId,
216
+ onProgress,
217
+ maxQuestions: preset.cove_max_questions || 5,
218
+ }
219
+ )
220
+ coveDirective = coveGateResult.coveDirective
221
+ }
222
+
223
+ // Pre-Stage 5.9: Adversarial White-Hat Security Fuzzing Gate (#302)
224
+ let securityDirective = null
225
+ let securityGateResult = null
226
+ if (preset?.security_gate_enabled) {
227
+ securityGateResult = runSecurityAuditGate(
228
+ syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
229
+ {
230
+ ctx,
231
+ runId,
232
+ onProgress,
233
+ strictVeto: preset.security_gate_strict_veto !== false,
234
+ }
235
+ )
236
+ securityDirective = securityGateResult.securityDirective
237
+ }
238
+
239
+ // Pre-Stage 5.10: Runtime CPU & Memory Profiler Gate (#315)
240
+ let profilerDirective = null
241
+ let profilerGateResult = null
242
+ if (preset?.profiler_gate_enabled) {
243
+ profilerGateResult = runProfilerGate(
244
+ syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
245
+ {
246
+ ctx,
247
+ runId,
248
+ onProgress,
249
+ iterations: preset.profiler_iterations || 100,
250
+ timeoutMs: preset.profiler_timeout_ms || 2000,
251
+ }
252
+ )
253
+ profilerDirective = profilerGateResult.profilerDirective
254
+ }
255
+
256
+ // Pre-Stage 5.11: Mutation Testing & Fault Injection Gate (#316)
257
+ let mutationDirective = null
258
+ let mutationGateResult = null
259
+ if (preset?.mutation_testing_enabled) {
260
+ mutationGateResult = runMutationGate(
261
+ syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
262
+ {
263
+ ctx,
264
+ runId,
265
+ onProgress,
266
+ minMutationScore: preset.min_mutation_score ?? 0.70,
267
+ }
268
+ )
269
+ mutationDirective = mutationGateResult.mutationDirective
270
+ }
271
+
202
272
  // Pre-Stage 6: Dynamic Consensus Threshold & Multi-Judge Escalation (#300)
203
273
  const escalationResult = evaluateJudgeEscalation({
204
274
  candidateOutputs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
@@ -249,7 +319,7 @@ export async function runMoAPipeline({
249
319
  preset: effectivePreset,
250
320
  successfulRefs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
251
321
  referenceOutputs, primaryJudge: effectivePrimaryJudge,
252
- userPrompt, judgeCriteria: ctxStage.judgeCriteria, prices, maxTokens: ctxStage.maxTokens,
322
+ userPrompt, judgeCriteria: [ctxStage.judgeCriteria, coveDirective, securityDirective, profilerDirective, mutationDirective].filter(Boolean).join('\n\n'), prices, maxTokens: ctxStage.maxTokens,
253
323
  callLlm, signal, onProgress, cwd, runId, isRefinement: ctxStage.isRefinement,
254
324
  isFastMode: ctxStage.isFastMode, collectedCtx: ctxStage.collectedCtx, startTime,
255
325
  })
@@ -307,7 +377,7 @@ export async function runMoAPipeline({
307
377
  }
308
378
  }
309
379
 
310
- const fileScoringDirective = [rawFileDirective, visualResult?.directive, adversarialTddResult?.directive, syntaxDirective, pruningDirective, ctxStage.stackAwareDirective, ctxStage.developerPreferencesDirective].filter(Boolean).join("\n\n") || null
380
+ const fileScoringDirective = [rawFileDirective, visualResult?.directive, adversarialTddResult?.directive, syntaxDirective, coveDirective, securityDirective, profilerDirective, mutationDirective, pruningDirective, ctxStage.stackAwareDirective, ctxStage.developerPreferencesDirective].filter(Boolean).join("\n\n") || null
311
381
 
312
382
  // Stage 7: Synthesis Phase
313
383
  const synthRes = await stageSynthesis({
@@ -0,0 +1,270 @@
1
+ /**
2
+ * DeepSeek Harness Mixture of Agents (MoA) — Runtime CPU & Memory Profiler Gate (#315)
3
+ *
4
+ * Micro-benchmarks candidate code execution duration, ops/sec throughput, and heap memory usage
5
+ * in an isolated sandbox with hard timeout protection to guide Judge selection towards
6
+ * algorithmically optimal solutions.
7
+ */
8
+
9
+ import vm from 'node:vm'
10
+ import { MOA_EVENTS, emitEvent } from '../events.js'
11
+
12
+ /**
13
+ * Extracts executable JavaScript code from markdown fences or raw code.
14
+ *
15
+ * @param {string} text
16
+ * @returns {string}
17
+ */
18
+ export function extractExecutableCode(text) {
19
+ if (!text || typeof text !== 'string') return ''
20
+ const jsFenceMatch = text.match(/```(?:javascript|js|node)\s*\n([\s\S]*?)\n```/i)
21
+ if (jsFenceMatch && jsFenceMatch[1]) {
22
+ return jsFenceMatch[1]
23
+ }
24
+ const anyFenceMatch = text.match(/```\w*\s*\n([\s\S]*?)\n```/)
25
+ if (anyFenceMatch && anyFenceMatch[1]) {
26
+ return anyFenceMatch[1]
27
+ }
28
+ return text
29
+ }
30
+
31
+ /**
32
+ * Profiles code execution in an isolated VM sandbox.
33
+ *
34
+ * @param {string} rawCode - Candidate source code or markdown
35
+ * @param {object} [options]
36
+ * @returns {object} Profiler report
37
+ */
38
+ export function profileCodeExecution(rawCode, options = {}) {
39
+ const {
40
+ iterations = 100,
41
+ timeoutMs = 2000,
42
+ } = options
43
+
44
+ const code = extractExecutableCode(rawCode)
45
+ if (!code || !code.trim()) {
46
+ return {
47
+ passed: false,
48
+ durationMs: 0,
49
+ heapBytesUsed: 0,
50
+ opsPerSec: 0,
51
+ iterations: 0,
52
+ error: 'Empty or unexecutable code provided for profiling.',
53
+ summary: 'Profiler skipped: no executable code.',
54
+ }
55
+ }
56
+
57
+ // Initial sandbox built-ins to ignore when scanning for user functions
58
+ const initialKeys = new Set([
59
+ 'console', 'Math', 'Date', 'Array', 'Object', 'String', 'Number',
60
+ 'Boolean', 'Set', 'Map', 'JSON', 'parseInt', 'parseFloat',
61
+ 'globalThis', 'global', '__mainFn', '__benchFn', '__benchArg'
62
+ ])
63
+
64
+ const sandbox = {
65
+ console: { log: () => {}, warn: () => {}, error: () => {} },
66
+ Math,
67
+ Date,
68
+ Array,
69
+ Object,
70
+ String,
71
+ Number,
72
+ Boolean,
73
+ Set,
74
+ Map,
75
+ JSON,
76
+ parseInt,
77
+ parseFloat,
78
+ }
79
+ sandbox.globalThis = sandbox
80
+ sandbox.global = sandbox
81
+
82
+ // Convert export declarations to var / global assignments
83
+ const cleanCode = code
84
+ .replace(/^\s*export\s+default\s+/gm, 'globalThis.__mainFn = ')
85
+ .replace(/^\s*export\s+(?:async\s+)?function\s+([a-zA-Z0-9_]+)/gm, 'function $1')
86
+ .replace(/^\s*export\s+const\s+([a-zA-Z0-9_]+)/gm, 'var $1')
87
+ .replace(/^\s*import\s+.*?;?\s*$/gm, '// [import stripped for benchmark]')
88
+
89
+ try {
90
+ const context = vm.createContext(sandbox)
91
+ // Run candidate code once to evaluate top-level declarations
92
+ const setupScript = new vm.Script(cleanCode)
93
+ setupScript.runInContext(context, { timeout: Math.min(1000, timeoutMs) })
94
+
95
+ // Find entrypoint function: either __mainFn or any user-defined function on sandbox
96
+ let targetFn = sandbox.__mainFn
97
+ if (typeof targetFn !== 'function') {
98
+ const userKeys = Object.keys(sandbox).filter((k) => !initialKeys.has(k) && typeof sandbox[k] === 'function')
99
+ if (userKeys.length > 0) {
100
+ targetFn = sandbox[userKeys[userKeys.length - 1]]
101
+ }
102
+ }
103
+
104
+ sandbox.__benchFn = targetFn || null
105
+ sandbox.__benchArg = 10
106
+
107
+ let benchScript
108
+ if (typeof sandbox.__benchFn === 'function') {
109
+ benchScript = new vm.Script(`
110
+ try {
111
+ __benchFn();
112
+ } catch (_) {
113
+ try { __benchFn(__benchArg); } catch (__) {}
114
+ }
115
+ `)
116
+ } else {
117
+ benchScript = new vm.Script(`(() => { ${cleanCode} })()`)
118
+ }
119
+
120
+ // Warm-up run
121
+ benchScript.runInContext(context, { timeout: Math.min(1000, timeoutMs) })
122
+
123
+ // Measure memory and time
124
+ const startMem = process.memoryUsage().heapUsed
125
+ const startHr = process.hrtime.bigint()
126
+
127
+ for (let i = 0; i < iterations; i++) {
128
+ benchScript.runInContext(context, { timeout: Math.min(500, timeoutMs) })
129
+ }
130
+
131
+ const endHr = process.hrtime.bigint()
132
+ const endMem = process.memoryUsage().heapUsed
133
+
134
+ const totalNs = Number(endHr - startHr)
135
+ const durationMs = Number((totalNs / 1_000_000).toFixed(3))
136
+ const avgLatencyMs = Number((durationMs / iterations).toFixed(4))
137
+ const heapBytesUsed = Math.max(0, endMem - startMem)
138
+ const opsPerSec = durationMs > 0 ? Math.round((iterations / (durationMs / 1000))) : 0
139
+
140
+ return {
141
+ passed: true,
142
+ durationMs,
143
+ avgLatencyMs,
144
+ heapBytesUsed,
145
+ opsPerSec,
146
+ iterations,
147
+ error: null,
148
+ summary: `⚡ Profiler: ${iterations} iter in ${durationMs}ms (${opsPerSec.toLocaleString()} ops/sec, ~${Math.round(heapBytesUsed / 1024)}KB heap).`,
149
+ }
150
+ } catch (err) {
151
+ const isTimeout = err.code === 'ERR_SCRIPT_EXECUTION_TIMEOUT' || err.message?.includes('timed out')
152
+ return {
153
+ passed: false,
154
+ durationMs: isTimeout ? timeoutMs : 0,
155
+ avgLatencyMs: 0,
156
+ heapBytesUsed: 0,
157
+ opsPerSec: 0,
158
+ iterations: 0,
159
+ error: isTimeout ? `Execution timed out after ${timeoutMs}ms (possible infinite loop or catastrophic complexity)` : err.message,
160
+ summary: isTimeout ? `🚨 Profiler: TIMEOUT (${timeoutMs}ms exceeded)` : `⚠️ Profiler Error: ${err.message}`,
161
+ }
162
+ }
163
+ }
164
+
165
+ /**
166
+ * Runs runtime performance profiling across all candidate solutions.
167
+ *
168
+ * @param {Array<object>} candidates
169
+ * @param {object} [options]
170
+ * @returns {object} Profiler gate result
171
+ */
172
+ export function runProfilerGate(candidates, options = {}) {
173
+ const {
174
+ ctx,
175
+ runId,
176
+ onProgress,
177
+ iterations = 100,
178
+ timeoutMs = 2000,
179
+ } = options
180
+
181
+ const results = []
182
+ if (!Array.isArray(candidates) || candidates.length === 0) {
183
+ return {
184
+ results: [],
185
+ fastestCandidate: null,
186
+ profilerDirective: null,
187
+ }
188
+ }
189
+
190
+ for (let i = 0; i < candidates.length; i++) {
191
+ const cand = candidates[i]
192
+ const text = typeof cand === 'string' ? cand : cand.output || cand.text || cand.code || ''
193
+ const report = profileCodeExecution(text, { iterations, timeoutMs })
194
+
195
+ if (typeof cand === 'object' && cand !== null) {
196
+ cand.profilerReport = report
197
+ }
198
+
199
+ results.push({
200
+ candidateIndex: i,
201
+ model: cand?.model || `Candidate ${i + 1}`,
202
+ report,
203
+ })
204
+
205
+ if (ctx) {
206
+ emitEvent(ctx, MOA_EVENTS.PROFILER_COMPLETE, {
207
+ runId,
208
+ candidateIndex: i,
209
+ durationMs: report.durationMs,
210
+ heapBytesUsed: report.heapBytesUsed,
211
+ opsPerSec: report.opsPerSec,
212
+ passed: report.passed,
213
+ error: report.error,
214
+ })
215
+ }
216
+ }
217
+
218
+ // Sort passing candidates by throughput (opsPerSec) descending
219
+ const valid = results.filter((r) => r.report.passed).sort((a, b) => b.report.opsPerSec - a.report.opsPerSec)
220
+ const fastestCandidate = valid[0] || null
221
+
222
+ if (typeof onProgress === 'function' && results.length > 0) {
223
+ const summaryList = results.map((r) => `Ref ${r.candidateIndex + 1}: ${r.report.passed ? `${r.report.durationMs}ms` : 'TIMEOUT'}`).join(', ')
224
+ onProgress(`⚡ *Runtime Profiler Gate*: ${summaryList}\n`)
225
+ }
226
+
227
+ const profilerDirective = formatProfilerDirective(results)
228
+
229
+ return {
230
+ results,
231
+ fastestCandidate,
232
+ profilerDirective,
233
+ }
234
+ }
235
+
236
+ /**
237
+ * Formats profiler benchmarks into a clear prompt directive for the Judge.
238
+ *
239
+ * @param {Array<object>} results
240
+ * @returns {string|null}
241
+ */
242
+ export function formatProfilerDirective(results) {
243
+ if (!Array.isArray(results) || results.length === 0) return null
244
+
245
+ const lines = [
246
+ '### ⚡ RUNTIME PERFORMANCE & CPU/MEMORY PROFILER REPORT',
247
+ 'Objective micro-benchmarks across candidates (100 iterations in isolated sandbox):',
248
+ ]
249
+
250
+ for (const r of results) {
251
+ const label = `Candidate ${r.candidateIndex + 1}`
252
+ const rep = r.report
253
+ if (rep.passed) {
254
+ const kb = Math.round(rep.heapBytesUsed / 1024)
255
+ lines.push(
256
+ `- **${label}**: ⏱️ **${rep.durationMs}ms** total (~${rep.avgLatencyMs}ms/op) | **${rep.opsPerSec.toLocaleString()} ops/sec** | Heap: ~${kb} KB`
257
+ )
258
+ } else {
259
+ lines.push(
260
+ `- **${label}**: ❌ **PERFORMANCE FAILURE**: ${rep.error}`
261
+ )
262
+ }
263
+ }
264
+
265
+ lines.push(
266
+ '\n**Judge Performance Guideline**: Penalize solutions with excessive latency or timeout failures. Prefer implementations demonstrating clean algorithmic scalability and minimal memory footprint.'
267
+ )
268
+
269
+ return lines.join('\n')
270
+ }