@goodandready/dsh-moa 0.2.76 → 0.2.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/README.ru.md +4 -0
- package/README.zh.md +4 -0
- package/lib/client.js +102 -2
- package/lib/events.js +4 -0
- package/lib/moa-schema.js +24 -0
- package/lib/pipeline/candidates.js +20 -2
- package/lib/pipeline/mutation-tester.js +323 -0
- package/lib/pipeline/privacy-sanitizer.js +213 -0
- package/lib/pipeline/runner.js +66 -6
- package/lib/pipeline/runtime-profiler.js +270 -0
- package/lib/pipeline/speculative-verifier.js +212 -0
- package/package.json +1 -1
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DeepSeek Harness Mixture of Agents (MoA) — Hybrid Private Edge & Privacy Sanitizer (#306)
|
|
3
|
+
*
|
|
4
|
+
* Protects project confidentiality in hybrid deployments by scrubbing local absolute paths,
|
|
5
|
+
* internal private IPs, API tokens, JWTs, and proprietary secrets before forwarding prompts
|
|
6
|
+
* or candidate proposals to external Cloud Judges. Preserves unsanitized originals locally
|
|
7
|
+
* for workspace promotion.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { MOA_EVENTS, emitEvent } from '../events.js'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Common regex definitions for sensitive data redaction.
|
|
14
|
+
*/
|
|
15
|
+
export const SANITIZATION_PATTERNS = {
|
|
16
|
+
OPENAI_OR_GENERIC_KEY: /sk-[a-zA-Z0-9_\-]{20,}/g,
|
|
17
|
+
BEARER_TOKEN: /Bearer\s+[a-zA-Z0-9_\-\.]{20,}/gi,
|
|
18
|
+
JWT: /ey[A-Za-z0-9_\-]{15,}\.ey[A-Za-z0-9_\-]{15,}\.[A-Za-z0-9_\-]{10,}/g,
|
|
19
|
+
AWS_KEY: /AKIA[0-9A-Z]{16}/g,
|
|
20
|
+
PRIVATE_KEY: /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g,
|
|
21
|
+
SECRET_ASSIGNMENT: /(?<=(?:api[_-]?key|secret|password|token|auth)\s*[:=]\s*["'])[a-zA-Z0-9_\-\.\/+=]{8,}(?=["'])/gi,
|
|
22
|
+
UNIX_HOME_PATH: /(?:\/(?:home|Users)\/[a-zA-Z0-9_\-]+(?:\/[a-zA-Z0-9_\-\.]+)*)/g,
|
|
23
|
+
WIN_USER_PATH: /(?:[A-Za-z]:\\(?:Users|Documents and Settings)\\[a-zA-Z0-9_\-]+(?:\\[a-zA-Z0-9_\-\.]+)*)/g,
|
|
24
|
+
PRIVATE_IP: /\b(?:192\.168\.\d{1,3}\.\d{1,3}|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})\b/g,
|
|
25
|
+
EMAIL: /\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b/g,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Sanitizes a text string by redacting sensitive secrets, internal paths, and PII.
|
|
30
|
+
*
|
|
31
|
+
* @param {string} text
|
|
32
|
+
* @param {object} [options]
|
|
33
|
+
* @returns {{ sanitizedText: string, redactionCount: number, redactingEntries: Array<object> }}
|
|
34
|
+
*/
|
|
35
|
+
export function sanitizeSensitiveText(text, options = {}) {
|
|
36
|
+
if (!text || typeof text !== 'string') {
|
|
37
|
+
return { sanitizedText: text || '', redactionCount: 0, redactingEntries: [] }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const {
|
|
41
|
+
redactSecrets = true,
|
|
42
|
+
redactPaths = true,
|
|
43
|
+
redactIps = true,
|
|
44
|
+
redactEmails = true,
|
|
45
|
+
} = options
|
|
46
|
+
|
|
47
|
+
let result = text
|
|
48
|
+
let redactionCount = 0
|
|
49
|
+
const redactingEntries = []
|
|
50
|
+
|
|
51
|
+
// 1. Secrets & API Keys
|
|
52
|
+
if (redactSecrets) {
|
|
53
|
+
result = result.replace(SANITIZATION_PATTERNS.PRIVATE_KEY, () => {
|
|
54
|
+
redactionCount++
|
|
55
|
+
redactingEntries.push({ type: 'PRIVATE_KEY' })
|
|
56
|
+
return '[REDACTED_PRIVATE_KEY]'
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
result = result.replace(SANITIZATION_PATTERNS.OPENAI_OR_GENERIC_KEY, () => {
|
|
60
|
+
redactionCount++
|
|
61
|
+
redactingEntries.push({ type: 'API_KEY' })
|
|
62
|
+
return '[REDACTED_API_KEY]'
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
result = result.replace(SANITIZATION_PATTERNS.JWT, () => {
|
|
66
|
+
redactionCount++
|
|
67
|
+
redactingEntries.push({ type: 'JWT' })
|
|
68
|
+
return '[REDACTED_JWT]'
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
result = result.replace(SANITIZATION_PATTERNS.AWS_KEY, () => {
|
|
72
|
+
redactionCount++
|
|
73
|
+
redactingEntries.push({ type: 'AWS_KEY' })
|
|
74
|
+
return '[REDACTED_AWS_KEY]'
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
result = result.replace(SANITIZATION_PATTERNS.SECRET_ASSIGNMENT, () => {
|
|
78
|
+
redactionCount++
|
|
79
|
+
redactingEntries.push({ type: 'SECRET_VALUE' })
|
|
80
|
+
return '[REDACTED_SECRET]'
|
|
81
|
+
})
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// 2. Private Paths
|
|
85
|
+
if (redactPaths) {
|
|
86
|
+
result = result.replace(SANITIZATION_PATTERNS.UNIX_HOME_PATH, (match) => {
|
|
87
|
+
redactionCount++
|
|
88
|
+
const parts = match.split('/')
|
|
89
|
+
const subpath = parts.slice(3).join('/')
|
|
90
|
+
return subpath ? `/workspace/${subpath}` : '/workspace'
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
result = result.replace(SANITIZATION_PATTERNS.WIN_USER_PATH, (match) => {
|
|
94
|
+
redactionCount++
|
|
95
|
+
const parts = match.split('\\')
|
|
96
|
+
const subpath = parts.slice(3).join('\\')
|
|
97
|
+
return subpath ? `C:\\workspace\\${subpath}` : 'C:\\workspace'
|
|
98
|
+
})
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// 3. Private IP addresses
|
|
102
|
+
if (redactIps) {
|
|
103
|
+
result = result.replace(SANITIZATION_PATTERNS.PRIVATE_IP, () => {
|
|
104
|
+
redactionCount++
|
|
105
|
+
redactingEntries.push({ type: 'INTERNAL_IP' })
|
|
106
|
+
return '[REDACTED_INTERNAL_IP]'
|
|
107
|
+
})
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// 4. Email addresses
|
|
111
|
+
if (redactEmails) {
|
|
112
|
+
result = result.replace(SANITIZATION_PATTERNS.EMAIL, () => {
|
|
113
|
+
redactionCount++
|
|
114
|
+
redactingEntries.push({ type: 'EMAIL' })
|
|
115
|
+
return '[REDACTED_EMAIL]'
|
|
116
|
+
})
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
sanitizedText: result,
|
|
121
|
+
redactionCount,
|
|
122
|
+
redactingEntries,
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Sanitizes candidate solutions before forwarding them to an external Cloud Judge.
|
|
128
|
+
* Preserves the original unmodified code in candidate.originalOutput for safe local promotion.
|
|
129
|
+
*
|
|
130
|
+
* @param {Array<object>} candidates
|
|
131
|
+
* @param {object} [options]
|
|
132
|
+
* @returns {object} Sanitization result
|
|
133
|
+
*/
|
|
134
|
+
export function sanitizeCandidatesForCloudJudge(candidates, options = {}) {
|
|
135
|
+
const {
|
|
136
|
+
ctx,
|
|
137
|
+
runId,
|
|
138
|
+
onProgress,
|
|
139
|
+
redactSecrets = true,
|
|
140
|
+
redactPaths = true,
|
|
141
|
+
redactIps = true,
|
|
142
|
+
redactEmails = true,
|
|
143
|
+
} = options
|
|
144
|
+
|
|
145
|
+
if (!Array.isArray(candidates) || candidates.length === 0) {
|
|
146
|
+
return {
|
|
147
|
+
sanitizedCandidates: [],
|
|
148
|
+
totalRedactions: 0,
|
|
149
|
+
privacyDirective: null,
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
let totalRedactions = 0
|
|
154
|
+
const sanitizedCandidates = candidates.map((cand, idx) => {
|
|
155
|
+
const rawText = typeof cand === 'string' ? cand : cand.output || cand.text || cand.code || ''
|
|
156
|
+
const { sanitizedText, redactionCount, redactingEntries } = sanitizeSensitiveText(rawText, {
|
|
157
|
+
redactSecrets,
|
|
158
|
+
redactPaths,
|
|
159
|
+
redactIps,
|
|
160
|
+
redactEmails,
|
|
161
|
+
})
|
|
162
|
+
|
|
163
|
+
totalRedactions += redactionCount
|
|
164
|
+
|
|
165
|
+
if (typeof cand === 'object' && cand !== null) {
|
|
166
|
+
if (!cand.originalOutput) {
|
|
167
|
+
cand.originalOutput = rawText
|
|
168
|
+
}
|
|
169
|
+
cand.sanitizedOutput = sanitizedText
|
|
170
|
+
cand.redactionCount = redactionCount
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (ctx && redactionCount > 0) {
|
|
174
|
+
emitEvent(ctx, MOA_EVENTS.PRIVACY_SANITIZED, {
|
|
175
|
+
runId,
|
|
176
|
+
candidateIndex: idx,
|
|
177
|
+
model: cand?.model || `Candidate ${idx + 1}`,
|
|
178
|
+
redactionCount,
|
|
179
|
+
types: [...new Set(redactingEntries.map((e) => e.type))],
|
|
180
|
+
})
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
return typeof cand === 'object' && cand !== null
|
|
184
|
+
? { ...cand, output: sanitizedText, text: sanitizedText }
|
|
185
|
+
: sanitizedText
|
|
186
|
+
})
|
|
187
|
+
|
|
188
|
+
if (typeof onProgress === 'function' && totalRedactions > 0) {
|
|
189
|
+
onProgress(`🛡️ *Hybrid Edge Privacy*: Sanitized ${totalRedactions} sensitive token(s) (secrets, paths, IPs) before cloud judge.\n`)
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const privacyDirective = totalRedactions > 0 ? formatPrivacyDirective(totalRedactions) : null
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
sanitizedCandidates,
|
|
196
|
+
totalRedactions,
|
|
197
|
+
privacyDirective,
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Formats a privacy directive header for the Cloud Judge prompt.
|
|
203
|
+
*
|
|
204
|
+
* @param {number} totalRedactions
|
|
205
|
+
* @returns {string}
|
|
206
|
+
*/
|
|
207
|
+
export function formatPrivacyDirective(totalRedactions) {
|
|
208
|
+
return [
|
|
209
|
+
'### 🛡️ PRIVACY & PII SANITIZATION ACTIVE',
|
|
210
|
+
`Notice: ${totalRedactions} internal system path(s), private network IP(s), or secret token(s) have been sanitized with [REDACTED_*] or relative paths to preserve user confidentiality.`,
|
|
211
|
+
'Evaluate the candidate solutions based on their architecture, correctness, and logic.',
|
|
212
|
+
].join('\n')
|
|
213
|
+
}
|
package/lib/pipeline/runner.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { sanitizeCandidatesForCloudJudge } from './privacy-sanitizer.js'
|
|
1
2
|
import { evaluateJudgeEscalation } from './escalation.js'
|
|
2
3
|
import { calculateSelfConsistencyAgreement } from './self-consistency.js'
|
|
3
4
|
import { pruneCandidateEnsemble, formatPruningDirective } from './context-pruner.js'
|
|
@@ -20,6 +21,8 @@ import { applySyntaxFilter, formatSyntaxDirective } from './syntax-filter.js'
|
|
|
20
21
|
import { stageEarlyConsensus } from './early-consensus.js'
|
|
21
22
|
import { runSecurityAuditGate } from './security-fuzzer.js'
|
|
22
23
|
import { runCoveVerificationGate } from './cove-verifier.js'
|
|
24
|
+
import { runProfilerGate } from './runtime-profiler.js'
|
|
25
|
+
import { runMutationGate } from './mutation-tester.js'
|
|
23
26
|
import { emitEvent, MOA_EVENTS } from '../events.js'
|
|
24
27
|
|
|
25
28
|
export async function runMoAPipeline({
|
|
@@ -234,6 +237,63 @@ export async function runMoAPipeline({
|
|
|
234
237
|
securityDirective = securityGateResult.securityDirective
|
|
235
238
|
}
|
|
236
239
|
|
|
240
|
+
// Pre-Stage 5.10: Runtime CPU & Memory Profiler Gate (#315)
|
|
241
|
+
let profilerDirective = null
|
|
242
|
+
let profilerGateResult = null
|
|
243
|
+
if (preset?.profiler_gate_enabled) {
|
|
244
|
+
profilerGateResult = runProfilerGate(
|
|
245
|
+
syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
246
|
+
{
|
|
247
|
+
ctx,
|
|
248
|
+
runId,
|
|
249
|
+
onProgress,
|
|
250
|
+
iterations: preset.profiler_iterations || 100,
|
|
251
|
+
timeoutMs: preset.profiler_timeout_ms || 2000,
|
|
252
|
+
}
|
|
253
|
+
)
|
|
254
|
+
profilerDirective = profilerGateResult.profilerDirective
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Pre-Stage 5.11: Mutation Testing & Fault Injection Gate (#316)
|
|
258
|
+
let mutationDirective = null
|
|
259
|
+
let mutationGateResult = null
|
|
260
|
+
if (preset?.mutation_testing_enabled) {
|
|
261
|
+
mutationGateResult = runMutationGate(
|
|
262
|
+
syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
263
|
+
{
|
|
264
|
+
ctx,
|
|
265
|
+
runId,
|
|
266
|
+
onProgress,
|
|
267
|
+
minMutationScore: preset.min_mutation_score ?? 0.70,
|
|
268
|
+
}
|
|
269
|
+
)
|
|
270
|
+
mutationDirective = mutationGateResult.mutationDirective
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Pre-Stage 5.12: Hybrid Edge Privacy Sanitizer Gate (#306)
|
|
274
|
+
let privacyDirective = null
|
|
275
|
+
let candidatesForJudge = syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs
|
|
276
|
+
let sanitizedReferenceOutputs = referenceOutputs
|
|
277
|
+
if (preset?.hybrid_edge_privacy_enabled || preset?.cloud_judge_sanitization_enabled) {
|
|
278
|
+
const sanitizeResult = sanitizeCandidatesForCloudJudge(candidatesForJudge, {
|
|
279
|
+
ctx,
|
|
280
|
+
runId,
|
|
281
|
+
onProgress,
|
|
282
|
+
redactSecrets: preset.redact_secrets !== false,
|
|
283
|
+
redactPaths: preset.redact_paths !== false,
|
|
284
|
+
redactIps: preset.redact_ips !== false,
|
|
285
|
+
})
|
|
286
|
+
candidatesForJudge = sanitizeResult.sanitizedCandidates
|
|
287
|
+
privacyDirective = sanitizeResult.privacyDirective
|
|
288
|
+
sanitizedReferenceOutputs = referenceOutputs.map((r) => {
|
|
289
|
+
const matching = candidatesForJudge.find((c) => c.index === r.index || c.model === r.model || c.label === r.label)
|
|
290
|
+
if (matching && matching.sanitizedOutput) {
|
|
291
|
+
return { ...r, text: matching.sanitizedOutput, output: matching.sanitizedOutput }
|
|
292
|
+
}
|
|
293
|
+
return r
|
|
294
|
+
})
|
|
295
|
+
}
|
|
296
|
+
|
|
237
297
|
// Pre-Stage 6: Dynamic Consensus Threshold & Multi-Judge Escalation (#300)
|
|
238
298
|
const escalationResult = evaluateJudgeEscalation({
|
|
239
299
|
candidateOutputs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
|
|
@@ -282,9 +342,9 @@ export async function runMoAPipeline({
|
|
|
282
342
|
// Stage 6: Multi-Judge Panel
|
|
283
343
|
const mjRes = await stageMultiJudge({
|
|
284
344
|
preset: effectivePreset,
|
|
285
|
-
successfulRefs:
|
|
286
|
-
referenceOutputs, primaryJudge: effectivePrimaryJudge,
|
|
287
|
-
userPrompt, judgeCriteria: [ctxStage.judgeCriteria, coveDirective, securityDirective].filter(Boolean).join('\n\n'), prices, maxTokens: ctxStage.maxTokens,
|
|
345
|
+
successfulRefs: candidatesForJudge,
|
|
346
|
+
referenceOutputs: sanitizedReferenceOutputs, primaryJudge: effectivePrimaryJudge,
|
|
347
|
+
userPrompt, judgeCriteria: [ctxStage.judgeCriteria, coveDirective, securityDirective, profilerDirective, mutationDirective, privacyDirective].filter(Boolean).join('\n\n'), prices, maxTokens: ctxStage.maxTokens,
|
|
288
348
|
callLlm, signal, onProgress, cwd, runId, isRefinement: ctxStage.isRefinement,
|
|
289
349
|
isFastMode: ctxStage.isFastMode, collectedCtx: ctxStage.collectedCtx, startTime,
|
|
290
350
|
})
|
|
@@ -313,8 +373,8 @@ export async function runMoAPipeline({
|
|
|
313
373
|
: null
|
|
314
374
|
|
|
315
375
|
// Pre-Judge Stage 6.8: Cross-Candidate Unchanged Base Context Pruning (#305)
|
|
316
|
-
let synthCandidateRefs =
|
|
317
|
-
let synthReferenceOutputs =
|
|
376
|
+
let synthCandidateRefs = candidatesForJudge
|
|
377
|
+
let synthReferenceOutputs = sanitizedReferenceOutputs
|
|
318
378
|
let pruningDirective = null
|
|
319
379
|
if (ctxStage.contextPruningEnabled && ctxStage.collectedCtx?.files?.length > 0) {
|
|
320
380
|
const pruneRes = pruneCandidateEnsemble(synthCandidateRefs, ctxStage.collectedCtx.files, {
|
|
@@ -342,7 +402,7 @@ export async function runMoAPipeline({
|
|
|
342
402
|
}
|
|
343
403
|
}
|
|
344
404
|
|
|
345
|
-
const fileScoringDirective = [rawFileDirective, visualResult?.directive, adversarialTddResult?.directive, syntaxDirective, coveDirective, securityDirective, pruningDirective, ctxStage.stackAwareDirective, ctxStage.developerPreferencesDirective].filter(Boolean).join("\n\n") || null
|
|
405
|
+
const fileScoringDirective = [rawFileDirective, visualResult?.directive, adversarialTddResult?.directive, syntaxDirective, coveDirective, securityDirective, profilerDirective, mutationDirective, privacyDirective, pruningDirective, ctxStage.stackAwareDirective, ctxStage.developerPreferencesDirective].filter(Boolean).join("\n\n") || null
|
|
346
406
|
|
|
347
407
|
// Stage 7: Synthesis Phase
|
|
348
408
|
const synthRes = await stageSynthesis({
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DeepSeek Harness Mixture of Agents (MoA) — Runtime CPU & Memory Profiler Gate (#315)
|
|
3
|
+
*
|
|
4
|
+
* Micro-benchmarks candidate code execution duration, ops/sec throughput, and heap memory usage
|
|
5
|
+
* in an isolated sandbox with hard timeout protection to guide Judge selection towards
|
|
6
|
+
* algorithmically optimal solutions.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import vm from 'node:vm'
|
|
10
|
+
import { MOA_EVENTS, emitEvent } from '../events.js'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Extracts executable JavaScript code from markdown fences or raw code.
|
|
14
|
+
*
|
|
15
|
+
* @param {string} text
|
|
16
|
+
* @returns {string}
|
|
17
|
+
*/
|
|
18
|
+
export function extractExecutableCode(text) {
|
|
19
|
+
if (!text || typeof text !== 'string') return ''
|
|
20
|
+
const jsFenceMatch = text.match(/```(?:javascript|js|node)\s*\n([\s\S]*?)\n```/i)
|
|
21
|
+
if (jsFenceMatch && jsFenceMatch[1]) {
|
|
22
|
+
return jsFenceMatch[1]
|
|
23
|
+
}
|
|
24
|
+
const anyFenceMatch = text.match(/```\w*\s*\n([\s\S]*?)\n```/)
|
|
25
|
+
if (anyFenceMatch && anyFenceMatch[1]) {
|
|
26
|
+
return anyFenceMatch[1]
|
|
27
|
+
}
|
|
28
|
+
return text
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Profiles code execution in an isolated VM sandbox.
|
|
33
|
+
*
|
|
34
|
+
* @param {string} rawCode - Candidate source code or markdown
|
|
35
|
+
* @param {object} [options]
|
|
36
|
+
* @returns {object} Profiler report
|
|
37
|
+
*/
|
|
38
|
+
export function profileCodeExecution(rawCode, options = {}) {
|
|
39
|
+
const {
|
|
40
|
+
iterations = 100,
|
|
41
|
+
timeoutMs = 2000,
|
|
42
|
+
} = options
|
|
43
|
+
|
|
44
|
+
const code = extractExecutableCode(rawCode)
|
|
45
|
+
if (!code || !code.trim()) {
|
|
46
|
+
return {
|
|
47
|
+
passed: false,
|
|
48
|
+
durationMs: 0,
|
|
49
|
+
heapBytesUsed: 0,
|
|
50
|
+
opsPerSec: 0,
|
|
51
|
+
iterations: 0,
|
|
52
|
+
error: 'Empty or unexecutable code provided for profiling.',
|
|
53
|
+
summary: 'Profiler skipped: no executable code.',
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Initial sandbox built-ins to ignore when scanning for user functions
|
|
58
|
+
const initialKeys = new Set([
|
|
59
|
+
'console', 'Math', 'Date', 'Array', 'Object', 'String', 'Number',
|
|
60
|
+
'Boolean', 'Set', 'Map', 'JSON', 'parseInt', 'parseFloat',
|
|
61
|
+
'globalThis', 'global', '__mainFn', '__benchFn', '__benchArg'
|
|
62
|
+
])
|
|
63
|
+
|
|
64
|
+
const sandbox = {
|
|
65
|
+
console: { log: () => {}, warn: () => {}, error: () => {} },
|
|
66
|
+
Math,
|
|
67
|
+
Date,
|
|
68
|
+
Array,
|
|
69
|
+
Object,
|
|
70
|
+
String,
|
|
71
|
+
Number,
|
|
72
|
+
Boolean,
|
|
73
|
+
Set,
|
|
74
|
+
Map,
|
|
75
|
+
JSON,
|
|
76
|
+
parseInt,
|
|
77
|
+
parseFloat,
|
|
78
|
+
}
|
|
79
|
+
sandbox.globalThis = sandbox
|
|
80
|
+
sandbox.global = sandbox
|
|
81
|
+
|
|
82
|
+
// Convert export declarations to var / global assignments
|
|
83
|
+
const cleanCode = code
|
|
84
|
+
.replace(/^\s*export\s+default\s+/gm, 'globalThis.__mainFn = ')
|
|
85
|
+
.replace(/^\s*export\s+(?:async\s+)?function\s+([a-zA-Z0-9_]+)/gm, 'function $1')
|
|
86
|
+
.replace(/^\s*export\s+const\s+([a-zA-Z0-9_]+)/gm, 'var $1')
|
|
87
|
+
.replace(/^\s*import\s+.*?;?\s*$/gm, '// [import stripped for benchmark]')
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
const context = vm.createContext(sandbox)
|
|
91
|
+
// Run candidate code once to evaluate top-level declarations
|
|
92
|
+
const setupScript = new vm.Script(cleanCode)
|
|
93
|
+
setupScript.runInContext(context, { timeout: Math.min(1000, timeoutMs) })
|
|
94
|
+
|
|
95
|
+
// Find entrypoint function: either __mainFn or any user-defined function on sandbox
|
|
96
|
+
let targetFn = sandbox.__mainFn
|
|
97
|
+
if (typeof targetFn !== 'function') {
|
|
98
|
+
const userKeys = Object.keys(sandbox).filter((k) => !initialKeys.has(k) && typeof sandbox[k] === 'function')
|
|
99
|
+
if (userKeys.length > 0) {
|
|
100
|
+
targetFn = sandbox[userKeys[userKeys.length - 1]]
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
sandbox.__benchFn = targetFn || null
|
|
105
|
+
sandbox.__benchArg = 10
|
|
106
|
+
|
|
107
|
+
let benchScript
|
|
108
|
+
if (typeof sandbox.__benchFn === 'function') {
|
|
109
|
+
benchScript = new vm.Script(`
|
|
110
|
+
try {
|
|
111
|
+
__benchFn();
|
|
112
|
+
} catch (_) {
|
|
113
|
+
try { __benchFn(__benchArg); } catch (__) {}
|
|
114
|
+
}
|
|
115
|
+
`)
|
|
116
|
+
} else {
|
|
117
|
+
benchScript = new vm.Script(`(() => { ${cleanCode} })()`)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Warm-up run
|
|
121
|
+
benchScript.runInContext(context, { timeout: Math.min(1000, timeoutMs) })
|
|
122
|
+
|
|
123
|
+
// Measure memory and time
|
|
124
|
+
const startMem = process.memoryUsage().heapUsed
|
|
125
|
+
const startHr = process.hrtime.bigint()
|
|
126
|
+
|
|
127
|
+
for (let i = 0; i < iterations; i++) {
|
|
128
|
+
benchScript.runInContext(context, { timeout: Math.min(500, timeoutMs) })
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const endHr = process.hrtime.bigint()
|
|
132
|
+
const endMem = process.memoryUsage().heapUsed
|
|
133
|
+
|
|
134
|
+
const totalNs = Number(endHr - startHr)
|
|
135
|
+
const durationMs = Number((totalNs / 1_000_000).toFixed(3))
|
|
136
|
+
const avgLatencyMs = Number((durationMs / iterations).toFixed(4))
|
|
137
|
+
const heapBytesUsed = Math.max(0, endMem - startMem)
|
|
138
|
+
const opsPerSec = durationMs > 0 ? Math.round((iterations / (durationMs / 1000))) : 0
|
|
139
|
+
|
|
140
|
+
return {
|
|
141
|
+
passed: true,
|
|
142
|
+
durationMs,
|
|
143
|
+
avgLatencyMs,
|
|
144
|
+
heapBytesUsed,
|
|
145
|
+
opsPerSec,
|
|
146
|
+
iterations,
|
|
147
|
+
error: null,
|
|
148
|
+
summary: `⚡ Profiler: ${iterations} iter in ${durationMs}ms (${opsPerSec.toLocaleString()} ops/sec, ~${Math.round(heapBytesUsed / 1024)}KB heap).`,
|
|
149
|
+
}
|
|
150
|
+
} catch (err) {
|
|
151
|
+
const isTimeout = err.code === 'ERR_SCRIPT_EXECUTION_TIMEOUT' || err.message?.includes('timed out')
|
|
152
|
+
return {
|
|
153
|
+
passed: false,
|
|
154
|
+
durationMs: isTimeout ? timeoutMs : 0,
|
|
155
|
+
avgLatencyMs: 0,
|
|
156
|
+
heapBytesUsed: 0,
|
|
157
|
+
opsPerSec: 0,
|
|
158
|
+
iterations: 0,
|
|
159
|
+
error: isTimeout ? `Execution timed out after ${timeoutMs}ms (possible infinite loop or catastrophic complexity)` : err.message,
|
|
160
|
+
summary: isTimeout ? `🚨 Profiler: TIMEOUT (${timeoutMs}ms exceeded)` : `⚠️ Profiler Error: ${err.message}`,
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Runs runtime performance profiling across all candidate solutions.
|
|
167
|
+
*
|
|
168
|
+
* @param {Array<object>} candidates
|
|
169
|
+
* @param {object} [options]
|
|
170
|
+
* @returns {object} Profiler gate result
|
|
171
|
+
*/
|
|
172
|
+
export function runProfilerGate(candidates, options = {}) {
|
|
173
|
+
const {
|
|
174
|
+
ctx,
|
|
175
|
+
runId,
|
|
176
|
+
onProgress,
|
|
177
|
+
iterations = 100,
|
|
178
|
+
timeoutMs = 2000,
|
|
179
|
+
} = options
|
|
180
|
+
|
|
181
|
+
const results = []
|
|
182
|
+
if (!Array.isArray(candidates) || candidates.length === 0) {
|
|
183
|
+
return {
|
|
184
|
+
results: [],
|
|
185
|
+
fastestCandidate: null,
|
|
186
|
+
profilerDirective: null,
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
for (let i = 0; i < candidates.length; i++) {
|
|
191
|
+
const cand = candidates[i]
|
|
192
|
+
const text = typeof cand === 'string' ? cand : cand.output || cand.text || cand.code || ''
|
|
193
|
+
const report = profileCodeExecution(text, { iterations, timeoutMs })
|
|
194
|
+
|
|
195
|
+
if (typeof cand === 'object' && cand !== null) {
|
|
196
|
+
cand.profilerReport = report
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
results.push({
|
|
200
|
+
candidateIndex: i,
|
|
201
|
+
model: cand?.model || `Candidate ${i + 1}`,
|
|
202
|
+
report,
|
|
203
|
+
})
|
|
204
|
+
|
|
205
|
+
if (ctx) {
|
|
206
|
+
emitEvent(ctx, MOA_EVENTS.PROFILER_COMPLETE, {
|
|
207
|
+
runId,
|
|
208
|
+
candidateIndex: i,
|
|
209
|
+
durationMs: report.durationMs,
|
|
210
|
+
heapBytesUsed: report.heapBytesUsed,
|
|
211
|
+
opsPerSec: report.opsPerSec,
|
|
212
|
+
passed: report.passed,
|
|
213
|
+
error: report.error,
|
|
214
|
+
})
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Sort passing candidates by throughput (opsPerSec) descending
|
|
219
|
+
const valid = results.filter((r) => r.report.passed).sort((a, b) => b.report.opsPerSec - a.report.opsPerSec)
|
|
220
|
+
const fastestCandidate = valid[0] || null
|
|
221
|
+
|
|
222
|
+
if (typeof onProgress === 'function' && results.length > 0) {
|
|
223
|
+
const summaryList = results.map((r) => `Ref ${r.candidateIndex + 1}: ${r.report.passed ? `${r.report.durationMs}ms` : 'TIMEOUT'}`).join(', ')
|
|
224
|
+
onProgress(`⚡ *Runtime Profiler Gate*: ${summaryList}\n`)
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const profilerDirective = formatProfilerDirective(results)
|
|
228
|
+
|
|
229
|
+
return {
|
|
230
|
+
results,
|
|
231
|
+
fastestCandidate,
|
|
232
|
+
profilerDirective,
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Formats profiler benchmarks into a clear prompt directive for the Judge.
|
|
238
|
+
*
|
|
239
|
+
* @param {Array<object>} results
|
|
240
|
+
* @returns {string|null}
|
|
241
|
+
*/
|
|
242
|
+
export function formatProfilerDirective(results) {
|
|
243
|
+
if (!Array.isArray(results) || results.length === 0) return null
|
|
244
|
+
|
|
245
|
+
const lines = [
|
|
246
|
+
'### ⚡ RUNTIME PERFORMANCE & CPU/MEMORY PROFILER REPORT',
|
|
247
|
+
'Objective micro-benchmarks across candidates (100 iterations in isolated sandbox):',
|
|
248
|
+
]
|
|
249
|
+
|
|
250
|
+
for (const r of results) {
|
|
251
|
+
const label = `Candidate ${r.candidateIndex + 1}`
|
|
252
|
+
const rep = r.report
|
|
253
|
+
if (rep.passed) {
|
|
254
|
+
const kb = Math.round(rep.heapBytesUsed / 1024)
|
|
255
|
+
lines.push(
|
|
256
|
+
`- **${label}**: ⏱️ **${rep.durationMs}ms** total (~${rep.avgLatencyMs}ms/op) | **${rep.opsPerSec.toLocaleString()} ops/sec** | Heap: ~${kb} KB`
|
|
257
|
+
)
|
|
258
|
+
} else {
|
|
259
|
+
lines.push(
|
|
260
|
+
`- **${label}**: ❌ **PERFORMANCE FAILURE**: ${rep.error}`
|
|
261
|
+
)
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
lines.push(
|
|
266
|
+
'\n**Judge Performance Guideline**: Penalize solutions with excessive latency or timeout failures. Prefer implementations demonstrating clean algorithmic scalability and minimal memory footprint.'
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
return lines.join('\n')
|
|
270
|
+
}
|