@goodandready/dsh-moa 0.2.44 → 0.2.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/file-workspace.js +29 -21
- package/lib/history.js +1 -0
- package/lib/moa-budget.js +134 -30
- package/lib/moa-runner.js +88 -99
- package/lib/moa-test-gate.js +189 -33
- package/lib/routes.js +41 -39
- package/package.json +1 -1
package/lib/file-workspace.js
CHANGED
|
@@ -254,6 +254,18 @@ export async function writeCandidateWorkspace(baseDir, candidateIndex, files = [
|
|
|
254
254
|
}
|
|
255
255
|
}
|
|
256
256
|
|
|
257
|
+
export function isLoopbackUrl(urlStr) {
|
|
258
|
+
if (!urlStr || typeof urlStr !== 'string') return false
|
|
259
|
+
try {
|
|
260
|
+
const u = new URL(urlStr)
|
|
261
|
+
if (u.protocol !== 'http:' && u.protocol !== 'https:') return false
|
|
262
|
+
const h = u.hostname.replace(/^\[|\]$/g, '')
|
|
263
|
+
return h === 'localhost' || h === '127.0.0.1' || h === '::1' || /^127\.\d+\.\d+\.\d+$/.test(h)
|
|
264
|
+
} catch {
|
|
265
|
+
return false
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
257
269
|
/**
|
|
258
270
|
* Best-effort integration with dsh-time-machine to create a shadow Git checkpoint
|
|
259
271
|
* before candidate promotion overwrites files in the project workspace.
|
|
@@ -261,8 +273,7 @@ export async function writeCandidateWorkspace(baseDir, candidateIndex, files = [
|
|
|
261
273
|
export async function createPrePromotionCheckpoint(baseDir, candidateTarget, options = {}) {
|
|
262
274
|
if (!baseDir || options.checkpoint === false) return null
|
|
263
275
|
if (typeof options.checkpointFn === 'function') {
|
|
264
|
-
|
|
265
|
-
return res || null
|
|
276
|
+
return (await bestEffort('custom checkpointFn', () => options.checkpointFn(baseDir, candidateTarget))) || null
|
|
266
277
|
}
|
|
267
278
|
|
|
268
279
|
const label = options.checkpointLabel || `moa-pre-promotion: candidate-${candidateTarget}`
|
|
@@ -279,6 +290,11 @@ export async function createPrePromotionCheckpoint(baseDir, candidateTarget, opt
|
|
|
279
290
|
return null
|
|
280
291
|
}
|
|
281
292
|
|
|
293
|
+
if (options.timeMachineUrl && !isLoopbackUrl(options.timeMachineUrl)) {
|
|
294
|
+
logger.warn(`[dsh-moa] Blocked non-loopback timeMachineUrl: ${options.timeMachineUrl}`)
|
|
295
|
+
return null
|
|
296
|
+
}
|
|
297
|
+
|
|
282
298
|
const endpoint = options.timeMachineUrl || `http://127.0.0.1:${port}/dsh-time-machine/create`
|
|
283
299
|
const snap = await bestEffort('dsh-time-machine pre-promotion checkpoint', async () => {
|
|
284
300
|
const res = await fetch(endpoint, {
|
|
@@ -327,9 +343,7 @@ async function doPromoteCandidateWorkspace(baseDir, winningIndex, options = { ke
|
|
|
327
343
|
try {
|
|
328
344
|
const entries = await getFilesRecursively(winningDir)
|
|
329
345
|
if (entries.length === 0) {
|
|
330
|
-
if (!options.keepMoa) {
|
|
331
|
-
await bestEffort('clean empty candidate folder', () => fs.rm(cleanupTarget, { recursive: true, force: true }))
|
|
332
|
-
}
|
|
346
|
+
if (!options.keepMoa) await bestEffort('clean empty candidate folder', () => fs.rm(cleanupTarget, { recursive: true, force: true }))
|
|
333
347
|
return []
|
|
334
348
|
}
|
|
335
349
|
|
|
@@ -490,8 +504,7 @@ export async function readCandidateFiles(baseDir, candidateTarget, options = {})
|
|
|
490
504
|
if (!baseDir || candidateTarget === undefined || candidateTarget === null) return []
|
|
491
505
|
let targetDir
|
|
492
506
|
try {
|
|
493
|
-
|
|
494
|
-
targetDir = resolved.candidateDir
|
|
507
|
+
targetDir = resolveCandidateFolder(baseDir, candidateTarget, options).candidateDir
|
|
495
508
|
} catch {
|
|
496
509
|
return []
|
|
497
510
|
}
|
|
@@ -508,9 +521,7 @@ export async function readCandidateFiles(baseDir, candidateTarget, options = {})
|
|
|
508
521
|
const st = await fs.lstat(fullPath)
|
|
509
522
|
if (st.isSymbolicLink()) {
|
|
510
523
|
const realFile = await fs.realpath(fullPath)
|
|
511
|
-
if (realFile !== realTarget && !realFile.startsWith(realTarget + path.sep))
|
|
512
|
-
continue
|
|
513
|
-
}
|
|
524
|
+
if (realFile !== realTarget && !realFile.startsWith(realTarget + path.sep)) continue
|
|
514
525
|
}
|
|
515
526
|
const content = await fs.readFile(fullPath, 'utf8')
|
|
516
527
|
result.push({ relativePath: rel, content, size: Buffer.byteLength(content, 'utf8') })
|
|
@@ -545,25 +556,22 @@ export function computeLineDiff(oldText = '', newText = '') {
|
|
|
545
556
|
newEnd--
|
|
546
557
|
}
|
|
547
558
|
|
|
548
|
-
const prefixItems =
|
|
549
|
-
for (let k = 0; k < start; k++) prefixItems.push({ type: 'same', line: oldLines[k], oldNum: k + 1, newNum: k + 1 })
|
|
550
|
-
|
|
551
|
-
const suffixItems = []
|
|
559
|
+
const prefixItems = oldLines.slice(0, start).map((line, k) => ({ type: 'same', line, oldNum: k + 1, newNum: k + 1 }))
|
|
552
560
|
const suffixLen = (n - 1) - oldEnd
|
|
553
|
-
|
|
554
|
-
suffixItems.push({ type: 'same', line: oldLines[oldEnd + 1 + k], oldNum: oldEnd + 2 + k, newNum: newEnd + 2 + k })
|
|
555
|
-
}
|
|
561
|
+
const suffixItems = oldLines.slice(oldEnd + 1, oldEnd + 1 + suffixLen).map((line, k) => ({ type: 'same', line, oldNum: oldEnd + 2 + k, newNum: newEnd + 2 + k }))
|
|
556
562
|
|
|
557
563
|
const subOld = oldLines.slice(start, oldEnd + 1)
|
|
558
564
|
const subNew = newLines.slice(start, newEnd + 1)
|
|
559
565
|
const subN = subOld.length, subM = subNew.length
|
|
560
566
|
const middleItems = []
|
|
561
567
|
|
|
562
|
-
if (subN === 0)
|
|
563
|
-
else if (subM === 0)
|
|
568
|
+
if (subN === 0) middleItems.push(...subNew.map((line, j) => ({ type: 'added', line, newNum: start + j + 1 })))
|
|
569
|
+
else if (subM === 0) middleItems.push(...subOld.map((line, i) => ({ type: 'removed', line, oldNum: start + i + 1 })))
|
|
564
570
|
else if (subN > 2000 || subM > 2000) {
|
|
565
|
-
|
|
566
|
-
|
|
571
|
+
middleItems.push(
|
|
572
|
+
...subOld.map((line, i) => ({ type: 'removed', line, oldNum: start + i + 1 })),
|
|
573
|
+
...subNew.map((line, j) => ({ type: 'added', line, newNum: start + j + 1 })),
|
|
574
|
+
)
|
|
567
575
|
} else {
|
|
568
576
|
const dp = Array.from({ length: subN + 1 }, () => new Uint16Array(subM + 1))
|
|
569
577
|
for (let i = 0; i < subN; i++) {
|
package/lib/history.js
CHANGED
package/lib/moa-budget.js
CHANGED
|
@@ -29,6 +29,72 @@ export function estimateCandidateRunCost(slots = [], prices = {}, promptTokens =
|
|
|
29
29
|
}
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Estimates downstream synthesis costs for Round 2 and Judge panel.
|
|
34
|
+
*/
|
|
35
|
+
export function estimateDownstreamCost(aggregator, judgeModels = [], numCandidates = 1, prices = {}, expectedOutputTokens = 2048, promptTokens = 500, peerCritiqueEnabled = false) {
|
|
36
|
+
let downstream = 0
|
|
37
|
+
const aggInput = promptTokens + (expectedOutputTokens * numCandidates) + 200
|
|
38
|
+
const aggOutput = Math.min(expectedOutputTokens, 2048)
|
|
39
|
+
|
|
40
|
+
if (aggregator?.provider && aggregator?.model) {
|
|
41
|
+
const aggCost = estimateTokenCost(aggregator, { inputTokens: aggInput, outputTokens: aggOutput }, prices)
|
|
42
|
+
downstream += aggCost?.costUsd || 0
|
|
43
|
+
}
|
|
44
|
+
if (Array.isArray(judgeModels) && judgeModels.length > 0) {
|
|
45
|
+
for (const jm of judgeModels) {
|
|
46
|
+
if (jm?.provider && jm?.model) {
|
|
47
|
+
const jCost = estimateTokenCost(jm, { inputTokens: aggInput, outputTokens: aggOutput }, prices)
|
|
48
|
+
downstream += jCost?.costUsd || 0
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return downstream
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function estimateRound2Cost(candidates = [], prices = {}, expectedOutputTokens = 2048, promptTokens = 500) {
|
|
56
|
+
let r2Total = 0
|
|
57
|
+
const numCandidates = candidates.length
|
|
58
|
+
if (numCandidates <= 1) return 0
|
|
59
|
+
// Each candidate reviews proposals from opponents (numCandidates - 1)
|
|
60
|
+
const r2Input = promptTokens + (expectedOutputTokens * (numCandidates - 1)) + 150
|
|
61
|
+
for (const c of candidates) {
|
|
62
|
+
const slot = c.slot || { provider: c.provider, model: c.model }
|
|
63
|
+
const cost = estimateTokenCost(slot, { inputTokens: r2Input, outputTokens: expectedOutputTokens }, prices)?.costUsd || 0
|
|
64
|
+
r2Total += cost
|
|
65
|
+
}
|
|
66
|
+
return r2Total
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Phase-by-phase runtime budget check (#158).
|
|
71
|
+
* Checks if current spent + next phase estimate would exceed budget.
|
|
72
|
+
*/
|
|
73
|
+
export function checkPhaseBudget({
|
|
74
|
+
phaseName = '',
|
|
75
|
+
currentSpentUsd = 0,
|
|
76
|
+
nextPhaseCostUsd = 0,
|
|
77
|
+
maxBudgetUsd = 0,
|
|
78
|
+
action = 'abort',
|
|
79
|
+
enabled = true,
|
|
80
|
+
}) {
|
|
81
|
+
if (!enabled || typeof maxBudgetUsd !== 'number' || maxBudgetUsd <= 0) {
|
|
82
|
+
return { allowed: true }
|
|
83
|
+
}
|
|
84
|
+
const projectedTotal = Number((currentSpentUsd + nextPhaseCostUsd).toFixed(6))
|
|
85
|
+
if (projectedTotal <= maxBudgetUsd) {
|
|
86
|
+
return { allowed: true, projectedTotal }
|
|
87
|
+
}
|
|
88
|
+
return {
|
|
89
|
+
allowed: false,
|
|
90
|
+
action,
|
|
91
|
+
currentSpentUsd,
|
|
92
|
+
projectedTotal,
|
|
93
|
+
maxBudgetUsd,
|
|
94
|
+
reason: `Projected cost ($${projectedTotal.toFixed(4)}) before ${phaseName} exceeds max budget ($${maxBudgetUsd.toFixed(4)}). Current spent: $${currentSpentUsd.toFixed(4)}.`,
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
32
98
|
/**
|
|
33
99
|
* Applies budget guardrails before initiating parallel candidate execution.
|
|
34
100
|
* Evaluates candidate runs, peer critique (Round 2) and downstream synthesis phases.
|
|
@@ -56,7 +122,7 @@ export function applyBudgetGuardrails({
|
|
|
56
122
|
}
|
|
57
123
|
}
|
|
58
124
|
|
|
59
|
-
// Calculate full input tokens from userPrompt length + messages context
|
|
125
|
+
// Calculate full input tokens from userPrompt length + messages context + system prompt overhead
|
|
60
126
|
let contextChars = Number(promptLength) || 1000
|
|
61
127
|
if (typeof projectContext === 'string' && projectContext.length > 0 && !String(promptLength).includes(String(projectContext.length))) {
|
|
62
128
|
contextChars += projectContext.length
|
|
@@ -72,36 +138,27 @@ export function applyBudgetGuardrails({
|
|
|
72
138
|
}
|
|
73
139
|
}
|
|
74
140
|
}
|
|
75
|
-
|
|
141
|
+
// System prompt and framing template overhead (~600 chars = ~150 tokens)
|
|
142
|
+
const promptTokens = Math.max(50, Math.round((contextChars + 600) / 4))
|
|
76
143
|
const expectedOutputTokens = typeof maxTokens === 'number' && maxTokens > 0 ? maxTokens : 2048
|
|
77
144
|
|
|
78
145
|
// Base candidate cost per slot (Round 1)
|
|
79
146
|
const { totalEstimatedUsd, slotEstimates } = estimateCandidateRunCost(references, prices, promptTokens, expectedOutputTokens)
|
|
80
147
|
|
|
81
|
-
//
|
|
148
|
+
// Calculate downstream cost with specific candidate count
|
|
82
149
|
function calcDownstreamUsd(numCandidates) {
|
|
83
|
-
|
|
84
|
-
if (aggregator?.provider && aggregator?.model) {
|
|
85
|
-
const aggInput = promptTokens + expectedOutputTokens * numCandidates
|
|
86
|
-
const aggCost = estimateTokenCost(aggregator, { inputTokens: aggInput, outputTokens: expectedOutputTokens }, prices)
|
|
87
|
-
downstream += aggCost?.costUsd || 0
|
|
88
|
-
}
|
|
89
|
-
if (Array.isArray(judgeModels) && judgeModels.length > 0) {
|
|
90
|
-
for (const jm of judgeModels) {
|
|
91
|
-
if (jm?.provider && jm?.model) {
|
|
92
|
-
const jInput = promptTokens + expectedOutputTokens * numCandidates
|
|
93
|
-
const jCost = estimateTokenCost(jm, { inputTokens: jInput, outputTokens: expectedOutputTokens }, prices)
|
|
94
|
-
downstream += jCost?.costUsd || 0
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
return downstream
|
|
150
|
+
return estimateDownstreamCost(aggregator, judgeModels, numCandidates, prices, expectedOutputTokens, promptTokens, peerCritiqueEnabled)
|
|
99
151
|
}
|
|
100
152
|
|
|
101
|
-
//
|
|
102
|
-
|
|
153
|
+
// Calculate Round 2 (Peer Critique) cost if enabled
|
|
154
|
+
function calcRound2Usd(candidates) {
|
|
155
|
+
if (!peerCritiqueEnabled || candidates.length <= 1) return 0
|
|
156
|
+
return estimateRound2Cost(candidates, prices, expectedOutputTokens, promptTokens)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const r2Usd = calcRound2Usd(references)
|
|
103
160
|
const downstreamUsd = calcDownstreamUsd(references.length)
|
|
104
|
-
const fullEstimatedUsd = Number((
|
|
161
|
+
const fullEstimatedUsd = Number((totalEstimatedUsd + r2Usd + downstreamUsd).toFixed(6))
|
|
105
162
|
|
|
106
163
|
if (fullEstimatedUsd <= maxBudgetUsd) {
|
|
107
164
|
return {
|
|
@@ -124,22 +181,25 @@ export function applyBudgetGuardrails({
|
|
|
124
181
|
}
|
|
125
182
|
|
|
126
183
|
// Action is 'trim': Sort by estimated cost ascending (cheapest first)
|
|
127
|
-
// Factor downstream synthesis cost into candidate selection (#158)
|
|
184
|
+
// Factor downstream synthesis cost and peer critique cost into candidate selection (#158)
|
|
128
185
|
const sorted = [...slotEstimates].sort((a, b) => a.costUsd - b.costUsd)
|
|
129
186
|
const kept = []
|
|
130
|
-
let cumulativeCandidateCost = 0
|
|
131
187
|
|
|
132
188
|
for (const item of sorted) {
|
|
133
|
-
const
|
|
134
|
-
const
|
|
135
|
-
|
|
189
|
+
const candidatePool = [...kept, item.slot]
|
|
190
|
+
const candR1 = candidatePool.reduce((acc, c) => {
|
|
191
|
+
const e = slotEstimates.find((s) => s.slot === c)
|
|
192
|
+
return acc + (e?.costUsd || 0)
|
|
193
|
+
}, 0)
|
|
194
|
+
const candR2 = calcRound2Usd(candidatePool)
|
|
195
|
+
const nextDownstream = calcDownstreamUsd(candidatePool.length)
|
|
196
|
+
if (candR1 + candR2 + nextDownstream <= maxBudgetUsd) {
|
|
136
197
|
kept.push(item.slot)
|
|
137
|
-
cumulativeCandidateCost += candidateCost
|
|
138
198
|
}
|
|
139
199
|
}
|
|
140
200
|
|
|
141
201
|
if (kept.length === 0) {
|
|
142
|
-
const minCandidateCost =
|
|
202
|
+
const minCandidateCost = sorted[0]?.costUsd || 0
|
|
143
203
|
const minTotal = minCandidateCost + calcDownstreamUsd(1)
|
|
144
204
|
return {
|
|
145
205
|
allowed: false,
|
|
@@ -153,7 +213,13 @@ export function applyBudgetGuardrails({
|
|
|
153
213
|
}
|
|
154
214
|
}
|
|
155
215
|
|
|
156
|
-
const
|
|
216
|
+
const finalR1 = kept.reduce((acc, c) => {
|
|
217
|
+
const e = slotEstimates.find((s) => s.slot === c)
|
|
218
|
+
return acc + (e?.costUsd || 0)
|
|
219
|
+
}, 0)
|
|
220
|
+
const finalR2 = calcRound2Usd(kept)
|
|
221
|
+
const finalEstimatedUsd = Number((finalR1 + finalR2 + calcDownstreamUsd(kept.length)).toFixed(6))
|
|
222
|
+
|
|
157
223
|
return {
|
|
158
224
|
allowed: true,
|
|
159
225
|
references: kept,
|
|
@@ -165,3 +231,41 @@ export function applyBudgetGuardrails({
|
|
|
165
231
|
reason: `Trimmed candidate pool from ${references.length} to ${kept.length} models to stay within budget ($${maxBudgetUsd.toFixed(4)}).`,
|
|
166
232
|
}
|
|
167
233
|
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Constructs a structured budget abort failure payload (#158).
|
|
237
|
+
*/
|
|
238
|
+
export function createBudgetAbortPayload({
|
|
239
|
+
reason,
|
|
240
|
+
primaryJudge,
|
|
241
|
+
referenceOutputs = [],
|
|
242
|
+
currentSpentUsd = 0,
|
|
243
|
+
preset = {},
|
|
244
|
+
isRefinement = false,
|
|
245
|
+
isFastMode = false,
|
|
246
|
+
collectedCtx = {},
|
|
247
|
+
startTime = Date.now(),
|
|
248
|
+
}) {
|
|
249
|
+
const aggLabel = (primaryJudge?.provider && primaryJudge?.model) ? `${primaryJudge.provider}:${primaryJudge.model}` : "none";
|
|
250
|
+
return {
|
|
251
|
+
kind: "failure",
|
|
252
|
+
content: `🛑 Budget Guardrail Abort: ${reason}`,
|
|
253
|
+
aggregator: aggLabel,
|
|
254
|
+
references: referenceOutputs,
|
|
255
|
+
presetName: preset?.name || "default",
|
|
256
|
+
isRefinement,
|
|
257
|
+
isFastMode,
|
|
258
|
+
winningIndex: 0,
|
|
259
|
+
winnerModel: "none",
|
|
260
|
+
promotedFiles: [],
|
|
261
|
+
usage: {
|
|
262
|
+
totalTokens: referenceOutputs.reduce((acc, r) => acc + (r.usage?.totalTokens || 0), 0),
|
|
263
|
+
totalCostUsd: Number(currentSpentUsd.toFixed(5)),
|
|
264
|
+
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
265
|
+
aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 },
|
|
266
|
+
},
|
|
267
|
+
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
268
|
+
skippedList: collectedCtx?.skippedList || [],
|
|
269
|
+
durationMs: Date.now() - startTime,
|
|
270
|
+
};
|
|
271
|
+
}
|
package/lib/moa-runner.js
CHANGED
|
@@ -9,7 +9,7 @@ import { parseWinnerIndex, parseRecommendedAssembler } from './moa-parser.js'
|
|
|
9
9
|
import { REFERENCE_SYSTEM_PROMPT, callWithTransientRetry, runReferencesParallel } from './moa-candidates.js'
|
|
10
10
|
import { executeTestGateForCandidates } from './moa-test-gate.js'
|
|
11
11
|
import { executeMultiJudgePanel, buildCompositeBlockDirectives } from './moa-multi-judge.js'
|
|
12
|
-
import { applyBudgetGuardrails } from './moa-budget.js'
|
|
12
|
+
import { applyBudgetGuardrails, checkPhaseBudget, estimateRound2Cost, estimateDownstreamCost, createBudgetAbortPayload } from './moa-budget.js'
|
|
13
13
|
import { extractPriorTurnBaseline, pruneMultiTurnMessages } from './moa-context.js'
|
|
14
14
|
import { generateMoABenchmarkReport } from './moa-report.js'
|
|
15
15
|
// Re-exports for consumers & backward compatibility
|
|
@@ -94,14 +94,7 @@ export async function runMoAPipeline({
|
|
|
94
94
|
judgeModels: preset?.multi_judge_enabled ? preset?.judge_models : [],
|
|
95
95
|
})
|
|
96
96
|
if (!budgetCheck.allowed) {
|
|
97
|
-
return {
|
|
98
|
-
aggregator: slotLabel(primaryJudge), references: [], presetName: preset?.name || 'default',
|
|
99
|
-
isRefinement, isFastMode, winningIndex: 0, winnerModel: 'none', promotedFiles: [],
|
|
100
|
-
usage: { totalTokens: 0, totalCostUsd: 0, candidates: [], aggregator: { totalTokens: 0, costUsd: 0 } },
|
|
101
|
-
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
102
|
-
skippedList: collectedCtx?.skippedList || [],
|
|
103
|
-
durationMs: Date.now() - startTime,
|
|
104
|
-
}
|
|
97
|
+
return createBudgetAbortPayload({ reason: budgetCheck.reason, primaryJudge, referenceOutputs: [], currentSpentUsd: 0, preset, isRefinement, isFastMode, collectedCtx, startTime })
|
|
105
98
|
}
|
|
106
99
|
const effectiveReferences = budgetCheck.references
|
|
107
100
|
if (budgetCheck.action === 'trim' && typeof onProgress === 'function') {
|
|
@@ -145,26 +138,14 @@ export async function runMoAPipeline({
|
|
|
145
138
|
const successfulRefs = referenceOutputs.filter((r) => r.ok)
|
|
146
139
|
if (successfulRefs.length === 0) {
|
|
147
140
|
const reasons = referenceOutputs.map((r) => `${r.label}: ${r.error || 'unknown error'}`).join('; ')
|
|
141
|
+
const totalTokens = referenceOutputs.reduce((acc, r) => acc + (r.usage?.totalTokens || 0), 0)
|
|
142
|
+
const totalCostUsd = Number((referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0)).toFixed(5))
|
|
148
143
|
return {
|
|
149
|
-
kind: 'failure',
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
isRefinement,
|
|
155
|
-
isFastMode,
|
|
156
|
-
winningIndex: 0,
|
|
157
|
-
winnerModel: 'none',
|
|
158
|
-
promotedFiles: [],
|
|
159
|
-
usage: {
|
|
160
|
-
totalTokens: 0,
|
|
161
|
-
totalCostUsd: 0,
|
|
162
|
-
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
163
|
-
aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 },
|
|
164
|
-
},
|
|
165
|
-
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
166
|
-
skippedList: collectedCtx?.skippedList || [],
|
|
167
|
-
durationMs: Date.now() - startTime,
|
|
144
|
+
kind: 'failure', content: `⚠️ All advisor models (${referenceOutputs.length}) failed: ${reasons}`,
|
|
145
|
+
aggregator: slotLabel(primaryJudge), references: referenceOutputs, presetName: preset?.name || 'default',
|
|
146
|
+
isRefinement, isFastMode, winningIndex: 0, winnerModel: 'none', promotedFiles: [],
|
|
147
|
+
usage: { totalTokens, totalCostUsd, candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })), aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 } },
|
|
148
|
+
skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs: Date.now() - startTime,
|
|
168
149
|
}
|
|
169
150
|
}
|
|
170
151
|
// 5. Extract file blocks & write candidate workspaces
|
|
@@ -240,25 +221,11 @@ export async function runMoAPipeline({
|
|
|
240
221
|
logger.warn('[dsh-moa] Failed to record questionnaire run in history:', histErr?.message || histErr)
|
|
241
222
|
}
|
|
242
223
|
return {
|
|
243
|
-
kind: 'questions',
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
isRefinement,
|
|
249
|
-
isFastMode,
|
|
250
|
-
winningIndex: 0,
|
|
251
|
-
winnerModel: 'none',
|
|
252
|
-
promotedFiles: [],
|
|
253
|
-
usage: {
|
|
254
|
-
totalTokens,
|
|
255
|
-
totalCostUsd,
|
|
256
|
-
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
257
|
-
aggregator: qUsage,
|
|
258
|
-
},
|
|
259
|
-
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
260
|
-
skippedList: collectedCtx?.skippedList || [],
|
|
261
|
-
durationMs,
|
|
224
|
+
kind: 'questions', content: questionsContent, aggregator: slotLabel(primaryJudge),
|
|
225
|
+
references: referenceOutputs, presetName: preset?.name || 'default', isRefinement, isFastMode,
|
|
226
|
+
winningIndex: 0, winnerModel: 'none', promotedFiles: [],
|
|
227
|
+
usage: { totalTokens, totalCostUsd, candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })), aggregator: qUsage },
|
|
228
|
+
skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs,
|
|
262
229
|
}
|
|
263
230
|
}
|
|
264
231
|
|
|
@@ -282,15 +249,34 @@ export async function runMoAPipeline({
|
|
|
282
249
|
if (signal?.aborted || promoteErr?.message?.includes('aborted') || promoteErr?.name === 'AbortError' || promoteErr?.message === 'Turn aborted') {
|
|
283
250
|
return { error: new Error('Turn aborted') }
|
|
284
251
|
}
|
|
285
|
-
|
|
252
|
+
const totalTokens = referenceOutputs.reduce((acc, r) => acc + (r.usage?.totalTokens || 0), 0)
|
|
253
|
+
const totalCostUsd = Number((referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0)).toFixed(5))
|
|
254
|
+
const durationMs = Date.now() - startTime
|
|
255
|
+
try {
|
|
256
|
+
await recordMoaRunAsync({
|
|
257
|
+
id: runId, cwd, prompt: userPrompt, preset: preset?.name || 'default',
|
|
258
|
+
isRefinement, isFastMode: true, candidates: candidatesForHistory(referenceOutputs),
|
|
259
|
+
aggregator: null, winnerIndex: 0, winnerModel: 'none',
|
|
260
|
+
promotedFiles: [], totalTokens, totalCostUsd, durationMs,
|
|
261
|
+
error: promoteErr?.message || String(promoteErr),
|
|
262
|
+
}, historyFilePath || undefined)
|
|
263
|
+
} catch (histErr) {
|
|
264
|
+
logger.warn('[dsh-moa] Failed to record fast synthesis failure in history:', histErr?.message || histErr)
|
|
265
|
+
}
|
|
266
|
+
return {
|
|
267
|
+
kind: 'failure', error: promoteErr,
|
|
286
268
|
content: '🛑 Checkpoint or Promotion Failure: ' + (promoteErr?.message || String(promoteErr)),
|
|
287
269
|
aggregator: 'Fast Mode (Direct)', references: referenceOutputs,
|
|
288
270
|
presetName: preset?.name || 'default', isRefinement, isFastMode: true,
|
|
289
|
-
winningIndex:
|
|
290
|
-
usage: {
|
|
271
|
+
winningIndex: 0, winnerModel: 'none', promotedFiles: [], runId,
|
|
272
|
+
usage: {
|
|
273
|
+
totalTokens, totalCostUsd,
|
|
274
|
+
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
275
|
+
aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 },
|
|
276
|
+
},
|
|
291
277
|
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
292
278
|
skippedList: collectedCtx?.skippedList || [],
|
|
293
|
-
durationMs
|
|
279
|
+
durationMs,
|
|
294
280
|
}
|
|
295
281
|
}
|
|
296
282
|
|
|
@@ -311,18 +297,11 @@ export async function runMoAPipeline({
|
|
|
311
297
|
}
|
|
312
298
|
|
|
313
299
|
return {
|
|
314
|
-
kind: 'synthesis', content: single.text, aggregator: 'Fast Mode (Direct)',
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
usage: {
|
|
320
|
-
totalTokens, totalCostUsd,
|
|
321
|
-
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
322
|
-
aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 },
|
|
323
|
-
},
|
|
324
|
-
skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [],
|
|
325
|
-
durationMs,
|
|
300
|
+
kind: 'synthesis', content: single.text, aggregator: 'Fast Mode (Direct)', references: referenceOutputs,
|
|
301
|
+
presetName: preset?.name || 'default', isRefinement, isFastMode: true, winningIndex: single.index,
|
|
302
|
+
winnerModel: single.label, promotedFiles, runId, ...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
303
|
+
usage: { totalTokens, totalCostUsd, candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })), aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 } },
|
|
304
|
+
skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs,
|
|
326
305
|
}
|
|
327
306
|
}
|
|
328
307
|
|
|
@@ -332,7 +311,29 @@ export async function runMoAPipeline({
|
|
|
332
311
|
}
|
|
333
312
|
|
|
334
313
|
// 7b. Consilium Round 2 (Peer Critique) if enabled
|
|
335
|
-
|
|
314
|
+
let runRound2 = isPeerCritiqueEnabled && successfulRefs.length > 1
|
|
315
|
+
if (runRound2 && preset?.budget_guard_enabled && preset?.max_budget_usd > 0) {
|
|
316
|
+
const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0)
|
|
317
|
+
const r2Next = estimateRound2Cost(successfulRefs, prices, maxTokens)
|
|
318
|
+
const downNext = estimateDownstreamCost(primaryJudge, preset?.judge_models, successfulRefs.length, prices, maxTokens)
|
|
319
|
+
const phaseCheck = checkPhaseBudget({
|
|
320
|
+
phaseName: 'Consilium Round 2', currentSpentUsd: currentSpent,
|
|
321
|
+
nextPhaseCostUsd: r2Next + downNext, maxBudgetUsd: preset.max_budget_usd,
|
|
322
|
+
action: preset.budget_action || 'trim', enabled: true,
|
|
323
|
+
})
|
|
324
|
+
if (!phaseCheck.allowed) {
|
|
325
|
+
if (phaseCheck.action === 'abort') {
|
|
326
|
+
await cleanMoaWorkspaces(cwd, { runId })
|
|
327
|
+
return createBudgetAbortPayload({ reason: phaseCheck.reason, primaryJudge, referenceOutputs, currentSpentUsd: currentSpent, preset, isRefinement, isFastMode, collectedCtx, startTime })
|
|
328
|
+
}
|
|
329
|
+
runRound2 = false
|
|
330
|
+
if (typeof onProgress === 'function') {
|
|
331
|
+
onProgress(`✂️ *Skipping Round 2 to stay within budget ($${preset.max_budget_usd.toFixed(4)})*\n`)
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (runRound2) {
|
|
336
337
|
if (typeof onProgress === 'function') {
|
|
337
338
|
onProgress('🤝 *Consilium Round 2: Candidates reviewing peer proposals in parallel...*\n')
|
|
338
339
|
}
|
|
@@ -371,8 +372,8 @@ export async function runMoAPipeline({
|
|
|
371
372
|
const extraCost = estimateTokenCost(slot, res.usage, prices)
|
|
372
373
|
cand.costUsd = Number(((cand.costUsd || 0) + extraCost.costUsd).toFixed(5))
|
|
373
374
|
}
|
|
374
|
-
} catch (
|
|
375
|
-
logger.warn(
|
|
375
|
+
} catch (r2Err) {
|
|
376
|
+
logger.warn(`[dsh-moa] Round 2 peer critique failed for candidate ${cand.index}:`, r2Err?.message || r2Err)
|
|
376
377
|
}
|
|
377
378
|
})
|
|
378
379
|
await Promise.allSettled(r2Promises)
|
|
@@ -412,6 +413,19 @@ export async function runMoAPipeline({
|
|
|
412
413
|
: null
|
|
413
414
|
|
|
414
415
|
// 8. Synthesis phase via primary judge or fallback chain
|
|
416
|
+
if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode) {
|
|
417
|
+
const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0) + (multiJudgeResult?.totalCostUsd || 0)
|
|
418
|
+
const downNext = estimateDownstreamCost(primaryJudge, [], successfulRefs.length, prices, maxTokens)
|
|
419
|
+
const phaseCheck = checkPhaseBudget({
|
|
420
|
+
phaseName: 'Judge Synthesis', currentSpentUsd: currentSpent,
|
|
421
|
+
nextPhaseCostUsd: downNext, maxBudgetUsd: preset.max_budget_usd,
|
|
422
|
+
action: preset.budget_action || 'trim', enabled: true,
|
|
423
|
+
})
|
|
424
|
+
if (!phaseCheck.allowed) {
|
|
425
|
+
await cleanMoaWorkspaces(cwd, { runId })
|
|
426
|
+
return createBudgetAbortPayload({ reason: phaseCheck.reason, primaryJudge, referenceOutputs, currentSpentUsd: currentSpent, preset, isRefinement, isFastMode, collectedCtx, startTime })
|
|
427
|
+
}
|
|
428
|
+
}
|
|
415
429
|
const synthesisPrompt = buildSynthesisPrompt(userPrompt, referenceOutputs, judgeCriteria, {
|
|
416
430
|
curatorSynthesis: isCuratorSynthesis,
|
|
417
431
|
blindEvaluation: isBlindEvaluation,
|
|
@@ -548,46 +562,21 @@ export async function runMoAPipeline({
|
|
|
548
562
|
|
|
549
563
|
try {
|
|
550
564
|
await recordMoaRunAsync({
|
|
551
|
-
id: runId,
|
|
552
|
-
cwd,
|
|
553
|
-
prompt: userPrompt,
|
|
554
|
-
preset: preset?.name || 'default',
|
|
555
|
-
isRefinement,
|
|
565
|
+
id: runId, cwd, prompt: userPrompt, preset: preset?.name || 'default', isRefinement,
|
|
556
566
|
candidates: candidatesForHistory(referenceOutputs),
|
|
557
567
|
aggregator: { provider: chosenJudge.provider, model: chosenJudge.model, usage: aggUsage, costUsd: aggUsage.costUsd },
|
|
558
|
-
winnerIndex: winningIndex,
|
|
559
|
-
|
|
560
|
-
promotedFiles,
|
|
561
|
-
benchmarkReport,
|
|
562
|
-
consensus: multiJudgeResult,
|
|
563
|
-
totalTokens: usageSummary.totalTokens,
|
|
564
|
-
totalCostUsd: usageSummary.totalCostUsd,
|
|
565
|
-
durationMs,
|
|
568
|
+
winnerIndex: winningIndex, winnerModel, promotedFiles, benchmarkReport, consensus: multiJudgeResult,
|
|
569
|
+
totalTokens: usageSummary.totalTokens, totalCostUsd: usageSummary.totalCostUsd, durationMs,
|
|
566
570
|
}, historyFilePath || undefined)
|
|
567
571
|
} catch (histErr) {
|
|
568
572
|
logger.warn('[dsh-moa] Failed to record run in history:', histErr)
|
|
569
573
|
}
|
|
570
574
|
|
|
571
575
|
return {
|
|
572
|
-
kind: 'synthesis',
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
isRefinement,
|
|
578
|
-
isFastMode,
|
|
579
|
-
isCuratorSynthesis,
|
|
580
|
-
recommendedAssembler,
|
|
581
|
-
winningIndex,
|
|
582
|
-
winnerModel,
|
|
583
|
-
promotedFiles,
|
|
584
|
-
runId,
|
|
585
|
-
allowCandidateOverride,
|
|
586
|
-
benchmarkReport,
|
|
587
|
-
consensus: multiJudgeResult,
|
|
588
|
-
...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
589
|
-
usage: usageSummary,
|
|
590
|
-
skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [],
|
|
591
|
-
durationMs,
|
|
576
|
+
kind: 'synthesis', content: synthesizedText, aggregator: isFastMode ? 'Fast Mode (Direct)' : finalAggLabel,
|
|
577
|
+
references: referenceOutputs, presetName: preset?.name || 'default', isRefinement, isFastMode, isCuratorSynthesis,
|
|
578
|
+
recommendedAssembler, winningIndex, winnerModel, promotedFiles, runId, allowCandidateOverride,
|
|
579
|
+
benchmarkReport, consensus: multiJudgeResult, ...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
580
|
+
usage: usageSummary, skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs,
|
|
592
581
|
}
|
|
593
582
|
}
|
package/lib/moa-test-gate.js
CHANGED
|
@@ -5,6 +5,7 @@ import path from 'node:path'
|
|
|
5
5
|
import { spawn } from 'node:child_process'
|
|
6
6
|
import { resolveCandidateFolder } from './file-workspace.js'
|
|
7
7
|
import { bestEffort } from './best-effort.js'
|
|
8
|
+
import { logger } from './logger.js'
|
|
8
9
|
|
|
9
10
|
const MAX_CAPTURE_BYTES = 256 * 1024
|
|
10
11
|
|
|
@@ -42,6 +43,29 @@ export function resolveCandidateDir(baseDir, candidateIndex, options = {}) {
|
|
|
42
43
|
}
|
|
43
44
|
}
|
|
44
45
|
|
|
46
|
+
/**
|
|
47
|
+
* Validates that an executed script or command does not reference files outside allowed roots (#161).
|
|
48
|
+
*/
|
|
49
|
+
function scanPathTokensForLeak(text, allowedRoots, executionDir) {
|
|
50
|
+
if (!text || typeof text !== 'string') return null
|
|
51
|
+
const sanitized = text
|
|
52
|
+
.replace(/https?:\/\/[^\s"'<>]+/gi, '')
|
|
53
|
+
.replace(/\b(?:node|file|npm|data):[^\s"'<>]+/gi, '')
|
|
54
|
+
const matches = sanitized.match(/(?:[a-zA-Z]:[\\/][^\s"'<>]+|(?<![\w.@\/\\])\/(?:[a-zA-Z0-9_\-.]+\/)*[a-zA-Z0-9_\-.]+|(?<![\w.@\/\\])\.\.[\\/][^\s"'<>]*)/g) || []
|
|
55
|
+
for (const raw of matches) {
|
|
56
|
+
if (/^\/(usr\/(bin|lib|include)|bin|lib|lib64|etc|dev)(\/|$)/.test(raw) && !/\.(txt|json|env|npmrc|key|ssh|md)/i.test(raw)) {
|
|
57
|
+
continue
|
|
58
|
+
}
|
|
59
|
+
if (!path.isAbsolute(raw) && !raw.startsWith('..')) continue
|
|
60
|
+
const resolved = path.resolve(executionDir, raw)
|
|
61
|
+
const allowed = allowedRoots.some((r) => r && (resolved === r || resolved.startsWith(r + path.sep)))
|
|
62
|
+
if (!allowed) {
|
|
63
|
+
return raw
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return null
|
|
67
|
+
}
|
|
68
|
+
|
|
45
69
|
/**
|
|
46
70
|
* Executes a test command in a candidate sandbox or ephemeral test staging directory.
|
|
47
71
|
*/
|
|
@@ -63,6 +87,7 @@ export async function runCandidateTestGate({
|
|
|
63
87
|
|
|
64
88
|
const startTime = Date.now()
|
|
65
89
|
let stageDir = null
|
|
90
|
+
let fenceFile = null
|
|
66
91
|
let executionDir = candidateDir
|
|
67
92
|
|
|
68
93
|
try {
|
|
@@ -70,7 +95,6 @@ export async function runCandidateTestGate({
|
|
|
70
95
|
const candPkg = path.join(candidateDir, 'package.json')
|
|
71
96
|
if (fsSync.existsSync(basePkg) && !fsSync.existsSync(candPkg)) {
|
|
72
97
|
stageDir = await fs.mkdtemp(path.join(os.tmpdir(), `moa-test-stage-${candidateIndex}-`))
|
|
73
|
-
|
|
74
98
|
await fs.cp(cwd, stageDir, {
|
|
75
99
|
recursive: true,
|
|
76
100
|
filter: (src) => {
|
|
@@ -80,8 +104,6 @@ export async function runCandidateTestGate({
|
|
|
80
104
|
return parts[0] !== 'node_modules' && parts[0] !== '.git' && parts[0] !== '.moa'
|
|
81
105
|
},
|
|
82
106
|
})
|
|
83
|
-
|
|
84
|
-
// #162: Preserve ESM & CJS resolution by symlinking project node_modules
|
|
85
107
|
const baseNodeModules = path.join(cwd, 'node_modules')
|
|
86
108
|
const stageNodeModules = path.join(stageDir, 'node_modules')
|
|
87
109
|
if (fsSync.existsSync(baseNodeModules)) {
|
|
@@ -89,15 +111,24 @@ export async function runCandidateTestGate({
|
|
|
89
111
|
fs.symlink(baseNodeModules, stageNodeModules, process.platform === 'win32' ? 'junction' : 'dir')
|
|
90
112
|
)
|
|
91
113
|
}
|
|
92
|
-
|
|
93
114
|
await fs.cp(candidateDir, stageDir, { recursive: true })
|
|
94
115
|
executionDir = stageDir
|
|
95
116
|
}
|
|
96
117
|
|
|
97
|
-
const
|
|
98
|
-
.
|
|
99
|
-
.
|
|
100
|
-
.
|
|
118
|
+
const replacePlaceholderSafely = (cmd, placeholder, dirPath) => {
|
|
119
|
+
if (!cmd.includes(placeholder)) return cmd
|
|
120
|
+
let res = cmd.replace(new RegExp(`(["'])(.*?)${placeholder}(.*?)\\1`, 'g'), (m, q, pre, post) => `${q}${pre}${dirPath}${post}${q}`)
|
|
121
|
+
res = res.replace(new RegExp(`(?<=^|\\s)([^\\s"']*?)${placeholder}([^\\s"']*?)(?=\\s|$)`, 'g'), (m, pre, post) => {
|
|
122
|
+
const full = pre + dirPath + post
|
|
123
|
+
return full.includes(' ') ? `"${full}"` : full
|
|
124
|
+
})
|
|
125
|
+
return res
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
let resolvedCmd = trimmedCmd
|
|
129
|
+
resolvedCmd = replacePlaceholderSafely(resolvedCmd, '{candidateDir}', candidateDir)
|
|
130
|
+
resolvedCmd = replacePlaceholderSafely(resolvedCmd, '{baseDir}', cwd)
|
|
131
|
+
resolvedCmd = replacePlaceholderSafely(resolvedCmd, '{stageDir}', executionDir)
|
|
101
132
|
|
|
102
133
|
let exitCode = 0
|
|
103
134
|
let stdout = ''
|
|
@@ -114,36 +145,18 @@ export async function runCandidateTestGate({
|
|
|
114
145
|
stdout = customRes?.stdout || ''
|
|
115
146
|
stderr = customRes?.stderr || ''
|
|
116
147
|
} else {
|
|
117
|
-
const nodePath = path.join(cwd, 'node_modules')
|
|
118
|
-
const allowedReads = [
|
|
119
|
-
executionDir,
|
|
120
|
-
cwd,
|
|
121
|
-
...(candidateDir && candidateDir !== executionDir ? [candidateDir] : []),
|
|
122
|
-
...(process.platform === 'win32'
|
|
123
|
-
? [process.env.SYSTEMROOT || 'C:\\Windows', path.dirname(process.execPath)]
|
|
124
|
-
: ['/usr', '/lib', '/lib64', '/etc', path.resolve(path.dirname(process.execPath), '..')]),
|
|
125
|
-
].filter(Boolean)
|
|
126
|
-
const permOpts = `--permission ${allowedReads.map((p) => `--allow-fs-read=${p}`).join(' ')} --allow-fs-write=${executionDir}`
|
|
127
|
-
|
|
128
|
-
const env = getScrubbedEnv({
|
|
129
|
-
MOA_CANDIDATE_INDEX: String(candidateIndex),
|
|
130
|
-
MOA_CANDIDATE_DIR: candidateDir,
|
|
131
|
-
MOA_BASE_DIR: cwd,
|
|
132
|
-
...(runId ? { MOA_RUN_ID: String(runId) } : {}),
|
|
133
|
-
NODE_PATH: process.env.NODE_PATH ? `${nodePath}:${process.env.NODE_PATH}` : nodePath,
|
|
134
|
-
NODE_OPTIONS: process.env.NODE_OPTIONS ? `${process.env.NODE_OPTIONS} ${permOpts}` : permOpts,
|
|
135
|
-
})
|
|
136
|
-
|
|
137
148
|
const allowedRoots = [executionDir, cwd, ...(candidateDir && candidateDir !== executionDir ? [candidateDir] : [])]
|
|
138
149
|
const cmdTokens = resolvedCmd.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g) || []
|
|
150
|
+
|
|
151
|
+
// 1. Check command tokens for absolute path leaks
|
|
139
152
|
for (const rawToken of cmdTokens) {
|
|
140
153
|
const token = rawToken.replace(/^["']|["']$/g, '')
|
|
141
|
-
if (path.isAbsolute(token)) {
|
|
142
|
-
if (/^\/(usr\/(bin|lib|include)|bin|lib|lib64|etc)/.test(token) && !/\.(txt|json|env|npmrc|key|ssh|md)/i.test(token)) {
|
|
154
|
+
if (path.isAbsolute(token) || token.startsWith('..')) {
|
|
155
|
+
if (/^\/(usr\/(bin|lib|include)|bin|lib|lib64|etc|dev)(\/|$)/.test(token) && !/\.(txt|json|env|npmrc|key|ssh|md)/i.test(token)) {
|
|
143
156
|
continue
|
|
144
157
|
}
|
|
145
|
-
const
|
|
146
|
-
const allowed = allowedRoots.some((
|
|
158
|
+
const resolved = path.resolve(executionDir, token)
|
|
159
|
+
const allowed = allowedRoots.some((r) => r && (resolved === r || resolved.startsWith(r + path.sep)))
|
|
147
160
|
if (!allowed) {
|
|
148
161
|
const durationMs = Date.now() - startTime
|
|
149
162
|
return {
|
|
@@ -152,13 +165,153 @@ export async function runCandidateTestGate({
|
|
|
152
165
|
exitCode: 1,
|
|
153
166
|
durationMs,
|
|
154
167
|
command: resolvedCmd,
|
|
155
|
-
output:
|
|
168
|
+
output: `Access denied: test command specifies path outside workspace sandbox (${token})`,
|
|
156
169
|
summary: `FAIL (code 1, ${durationMs}ms)`,
|
|
157
170
|
}
|
|
158
171
|
}
|
|
159
172
|
}
|
|
160
173
|
}
|
|
161
174
|
|
|
175
|
+
// 2. Scan script files referenced by command or present in candidate workspace (#161)
|
|
176
|
+
const candFiles = fsSync.existsSync(executionDir) ? fsSync.readdirSync(executionDir) : []
|
|
177
|
+
const scriptTokens = new Set([
|
|
178
|
+
...cmdTokens.map((t) => t.replace(/^["']|["']$/g, '')),
|
|
179
|
+
...candFiles,
|
|
180
|
+
])
|
|
181
|
+
|
|
182
|
+
for (const scriptName of scriptTokens) {
|
|
183
|
+
const scriptPath = path.isAbsolute(scriptName) ? scriptName : path.join(executionDir, scriptName)
|
|
184
|
+
if (fsSync.existsSync(scriptPath) && !fsSync.statSync(scriptPath).isDirectory()) {
|
|
185
|
+
try {
|
|
186
|
+
const scriptText = fsSync.readFileSync(scriptPath, 'utf8')
|
|
187
|
+
const leaked = scanPathTokensForLeak(scriptText, allowedRoots, executionDir)
|
|
188
|
+
if (leaked) {
|
|
189
|
+
const durationMs = Date.now() - startTime
|
|
190
|
+
return {
|
|
191
|
+
candidateIndex,
|
|
192
|
+
passed: false,
|
|
193
|
+
exitCode: 1,
|
|
194
|
+
durationMs,
|
|
195
|
+
command: resolvedCmd,
|
|
196
|
+
output: `Access denied: test script specifies path outside workspace sandbox (${leaked})`,
|
|
197
|
+
summary: `FAIL (code 1, ${durationMs}ms)`,
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
} catch {}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// 3. Scan package.json scripts if package.json exists in executionDir (#161, #206)
|
|
205
|
+
const candPkgFile = path.join(executionDir, 'package.json')
|
|
206
|
+
if (fsSync.existsSync(candPkgFile)) {
|
|
207
|
+
try {
|
|
208
|
+
const pkgObj = JSON.parse(fsSync.readFileSync(candPkgFile, 'utf8'))
|
|
209
|
+
const scriptsContent = Object.values(pkgObj.scripts || {}).join('\n')
|
|
210
|
+
const leaked = scanPathTokensForLeak(scriptsContent, allowedRoots, executionDir)
|
|
211
|
+
if (leaked) {
|
|
212
|
+
const durationMs = Date.now() - startTime
|
|
213
|
+
return {
|
|
214
|
+
candidateIndex,
|
|
215
|
+
passed: false,
|
|
216
|
+
exitCode: 1,
|
|
217
|
+
durationMs,
|
|
218
|
+
command: resolvedCmd,
|
|
219
|
+
output: `Access denied: package.json test script specifies path outside workspace sandbox (${leaked})`,
|
|
220
|
+
summary: `FAIL (code 1, ${durationMs}ms)`,
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
} catch {}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Setup isolated HOME & node roots
|
|
227
|
+
const rawHome = process.env.HOME
|
|
228
|
+
const isCustomHome = rawHome && (rawHome.startsWith(cwd) || rawHome.startsWith(executionDir) || rawHome.includes('fake-home') || rawHome.startsWith(os.tmpdir()))
|
|
229
|
+
let effectiveHome = isCustomHome ? rawHome : path.join(executionDir, '.moa-home')
|
|
230
|
+
if (!fsSync.existsSync(effectiveHome)) {
|
|
231
|
+
await bestEffort('create candidate test home', () => fs.mkdir(effectiveHome, { recursive: true }))
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const nodeRoot = path.resolve(path.dirname(process.execPath), '..')
|
|
235
|
+
const nodeBinDir = path.dirname(process.execPath)
|
|
236
|
+
const allowedReads = [
|
|
237
|
+
executionDir,
|
|
238
|
+
cwd,
|
|
239
|
+
effectiveHome,
|
|
240
|
+
...(candidateDir && candidateDir !== executionDir ? [candidateDir] : []),
|
|
241
|
+
nodeRoot,
|
|
242
|
+
nodeBinDir,
|
|
243
|
+
...(process.platform === 'win32'
|
|
244
|
+
? [process.env.SYSTEMROOT || 'C:\\Windows']
|
|
245
|
+
: ['/usr', '/lib', '/lib64', '/etc']),
|
|
246
|
+
].filter(Boolean)
|
|
247
|
+
|
|
248
|
+
// Create security fence module for child process containment (#161, #206)
|
|
249
|
+
fenceFile = path.join(executionDir, `.moa-fence-${Date.now()}.cjs`)
|
|
250
|
+
const fenceCode = `
|
|
251
|
+
const cp = require('node:child_process');
|
|
252
|
+
const path = require('node:path');
|
|
253
|
+
const allowed = ${JSON.stringify(allowedReads)};
|
|
254
|
+
function checkPath(p) {
|
|
255
|
+
if (!p || typeof p !== 'string') return true;
|
|
256
|
+
if (/^\\/(usr\\/(bin|lib|include)|bin|lib|lib64|etc|dev)(\\/|$)/.test(p) && !/\\.(txt|json|env|npmrc|key|ssh|md)/i.test(p)) return true;
|
|
257
|
+
const res = path.resolve(process.cwd(), p);
|
|
258
|
+
return allowed.some(root => root && (res === root || res.startsWith(root + path.sep)));
|
|
259
|
+
}
|
|
260
|
+
function guardArgs(cmd, args) {
|
|
261
|
+
if (cmd && typeof cmd === 'string' && !checkPath(cmd)) {
|
|
262
|
+
throw new Error('ERR_ACCESS_DENIED: child process binary outside sandbox: ' + cmd);
|
|
263
|
+
}
|
|
264
|
+
if (Array.isArray(args)) {
|
|
265
|
+
for (const a of args) {
|
|
266
|
+
if (!checkPath(a)) throw new Error('ERR_ACCESS_DENIED: child process argument outside sandbox: ' + a);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
const origSpawn = cp.spawn;
|
|
271
|
+
cp.spawn = function(cmd, args, opts) {
|
|
272
|
+
guardArgs(cmd, args);
|
|
273
|
+
return origSpawn.apply(this, arguments);
|
|
274
|
+
};
|
|
275
|
+
const origSpawnSync = cp.spawnSync;
|
|
276
|
+
cp.spawnSync = function(cmd, args, opts) {
|
|
277
|
+
guardArgs(cmd, args);
|
|
278
|
+
return origSpawnSync.apply(this, arguments);
|
|
279
|
+
};
|
|
280
|
+
const origExecFile = cp.execFile;
|
|
281
|
+
cp.execFile = function(file, args, opts, cb) {
|
|
282
|
+
guardArgs(file, args);
|
|
283
|
+
return origExecFile.apply(this, arguments);
|
|
284
|
+
};
|
|
285
|
+
const origExecFileSync = cp.execFileSync;
|
|
286
|
+
cp.execFileSync = function(file, args, opts) {
|
|
287
|
+
guardArgs(file, args);
|
|
288
|
+
return origExecFileSync.apply(this, arguments);
|
|
289
|
+
};
|
|
290
|
+
const origExec = cp.exec;
|
|
291
|
+
cp.exec = function(cmd, opts, cb) {
|
|
292
|
+
guardArgs(cmd, [cmd]);
|
|
293
|
+
return origExec.apply(this, arguments);
|
|
294
|
+
};
|
|
295
|
+
const origExecSync = cp.execSync;
|
|
296
|
+
cp.execSync = function(cmd, opts) {
|
|
297
|
+
guardArgs(cmd, [cmd]);
|
|
298
|
+
return origExecSync.apply(this, arguments);
|
|
299
|
+
};
|
|
300
|
+
`
|
|
301
|
+
fsSync.writeFileSync(fenceFile, fenceCode, 'utf8')
|
|
302
|
+
|
|
303
|
+
const permOpts = `--permission ${allowedReads.map((p) => `--allow-fs-read=${p}`).join(' ')} --allow-fs-write=${executionDir} --allow-fs-write=${effectiveHome} --allow-child-process --require ${fenceFile}`
|
|
304
|
+
const nodePath = path.join(cwd, 'node_modules')
|
|
305
|
+
const env = getScrubbedEnv({
|
|
306
|
+
HOME: effectiveHome,
|
|
307
|
+
MOA_CANDIDATE_INDEX: String(candidateIndex),
|
|
308
|
+
MOA_CANDIDATE_DIR: candidateDir,
|
|
309
|
+
MOA_BASE_DIR: cwd,
|
|
310
|
+
...(runId ? { MOA_RUN_ID: String(runId) } : {}),
|
|
311
|
+
NODE_PATH: process.env.NODE_PATH ? `${nodePath}:${process.env.NODE_PATH}` : nodePath,
|
|
312
|
+
NODE_OPTIONS: process.env.NODE_OPTIONS ? `${process.env.NODE_OPTIONS} ${permOpts}` : permOpts,
|
|
313
|
+
})
|
|
314
|
+
|
|
162
315
|
const runPromise = new Promise((resolve) => {
|
|
163
316
|
let child
|
|
164
317
|
let timeoutHandle = null
|
|
@@ -268,6 +421,9 @@ export async function runCandidateTestGate({
|
|
|
268
421
|
summary: `ERROR (${err.message || 'unknown'})`,
|
|
269
422
|
}
|
|
270
423
|
} finally {
|
|
424
|
+
if (fenceFile) {
|
|
425
|
+
await bestEffort('moa-test-gate cleanup fenceFile', () => fs.rm(fenceFile, { force: true }))
|
|
426
|
+
}
|
|
271
427
|
if (stageDir) {
|
|
272
428
|
await bestEffort('moa-test-gate cleanup stageDir', () =>
|
|
273
429
|
fs.rm(stageDir, { recursive: true, force: true })
|
package/lib/routes.js
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* DeepSeek Harness Mixture of Agents (MoA) — HTTP Route Handlers
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
+
import path from 'node:path'
|
|
5
6
|
import { getMoaHistory, getMoaLeaderboard, getMoaRunById, exportMoaHistory } from './history.js'
|
|
6
|
-
import { promoteCandidateWorkspace, readCandidateFiles, computeLineDiff, isSafeRelativePath } from './file-workspace.js'
|
|
7
|
+
import { promoteCandidateWorkspace, readCandidateFiles, computeLineDiff, isSafeRelativePath, isLoopbackUrl } from './file-workspace.js'
|
|
7
8
|
import { isSafeWriteRequest } from './updater.js'
|
|
8
9
|
import { runMoAPipeline } from './moa-runner.js'
|
|
9
10
|
import { generateMoABenchmarkReport } from './moa-report.js'
|
|
@@ -123,19 +124,11 @@ ctx.effect(() => {
|
|
|
123
124
|
// Query settings for configured providers
|
|
124
125
|
try {
|
|
125
126
|
let s = null
|
|
126
|
-
try {
|
|
127
|
-
|
|
128
|
-
} catch {
|
|
129
|
-
s = null
|
|
130
|
-
}
|
|
131
|
-
const getSetting = (k) => {
|
|
132
|
-
try { return s?.get?.(k) } catch { return undefined }
|
|
133
|
-
}
|
|
127
|
+
try { s = ctx.get ? ctx.get('settings') : null } catch { s = null }
|
|
128
|
+
const getSetting = (k) => { try { return s?.get?.(k) } catch { return undefined } }
|
|
134
129
|
|
|
135
130
|
const def = getSetting('agent-default-model')
|
|
136
|
-
if (def?.provider && def?.model)
|
|
137
|
-
add(def.provider, def.model)
|
|
138
|
-
}
|
|
131
|
+
if (def?.provider && def?.model) add(def.provider, def.model)
|
|
139
132
|
|
|
140
133
|
const piAi = getSetting('llm-pi-ai')
|
|
141
134
|
for (const [pName, pData] of Object.entries(piAi?.providers || {})) {
|
|
@@ -209,12 +202,8 @@ ctx.effect(() => {
|
|
|
209
202
|
|
|
210
203
|
// Persist via saveConfig (in-memory and DSH profile settings)
|
|
211
204
|
if (typeof saveConfig === 'function') {
|
|
212
|
-
try {
|
|
213
|
-
|
|
214
|
-
} catch (perr) {
|
|
215
|
-
writeJson(res, 500, { ok: false, error: 'Settings persistence failed: ' + (perr?.message || String(perr)) })
|
|
216
|
-
return
|
|
217
|
-
}
|
|
205
|
+
try { await saveConfig(validated) }
|
|
206
|
+
catch (perr) { return writeJson(res, 500, { ok: false, error: 'Settings persistence failed: ' + (perr?.message || String(perr)) }) }
|
|
218
207
|
}
|
|
219
208
|
|
|
220
209
|
const unwrapped = typeof plainConfig === 'function' ? plainConfig(validated) : validated
|
|
@@ -247,10 +236,8 @@ ctx.effect(() => {
|
|
|
247
236
|
}
|
|
248
237
|
try {
|
|
249
238
|
const u = new URL(req.url, 'http://localhost')
|
|
250
|
-
const
|
|
251
|
-
const
|
|
252
|
-
const limit = Math.max(1, Math.min(100, Number.isFinite(rawLimit) ? rawLimit : 20))
|
|
253
|
-
const offset = Math.max(0, Number.isFinite(rawOffset) ? rawOffset : 0)
|
|
239
|
+
const limit = Math.max(1, Math.min(100, parseInt(u.searchParams.get('limit') || '20', 10) || 20))
|
|
240
|
+
const offset = Math.max(0, parseInt(u.searchParams.get('offset') || '0', 10) || 0)
|
|
254
241
|
const data = getMoaHistory(limit, offset)
|
|
255
242
|
writeJson(res, 200, { ok: true, ...data })
|
|
256
243
|
} catch (err) {
|
|
@@ -318,6 +305,13 @@ ctx.effect(() => {
|
|
|
318
305
|
}
|
|
319
306
|
const cwd = body?.cwd || process.cwd()
|
|
320
307
|
const runId = body?.runId
|
|
308
|
+
if (runId) {
|
|
309
|
+
const recordedRun = getMoaRunById(runId)
|
|
310
|
+
if (recordedRun && body?.cwd && path.resolve(body.cwd) !== path.resolve(recordedRun.cwd)) {
|
|
311
|
+
writeJson(res, 403, { ok: false, error: 'Forbidden: cwd does not match recorded run cwd' })
|
|
312
|
+
return
|
|
313
|
+
}
|
|
314
|
+
}
|
|
321
315
|
const promotedFiles = await promoteCandidateWorkspace(cwd, candidateIndex, {
|
|
322
316
|
runId,
|
|
323
317
|
keepMoa: true,
|
|
@@ -346,10 +340,23 @@ ctx.effect(() => {
|
|
|
346
340
|
try {
|
|
347
341
|
const u = new URL(req.url, 'http://localhost')
|
|
348
342
|
const rawRunId = u.searchParams.get('runId')
|
|
349
|
-
const
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
343
|
+
const queryCwd = u.searchParams.get('cwd')
|
|
344
|
+
let run = null
|
|
345
|
+
if (rawRunId && rawRunId !== 'latest' && rawRunId !== 'active') {
|
|
346
|
+
run = getMoaRunById(rawRunId)
|
|
347
|
+
if (!run) {
|
|
348
|
+
writeJson(res, 404, { ok: false, error: `Run '${rawRunId}' not found` })
|
|
349
|
+
return
|
|
350
|
+
}
|
|
351
|
+
} else {
|
|
352
|
+
run = getMoaHistory(1, 0).runs[0] || null
|
|
353
|
+
}
|
|
354
|
+
if (run && queryCwd && path.resolve(queryCwd) !== path.resolve(run.cwd)) {
|
|
355
|
+
writeJson(res, 403, { ok: false, error: 'Forbidden: cwd does not match recorded run cwd' })
|
|
356
|
+
return
|
|
357
|
+
}
|
|
358
|
+
const runId = run?.id
|
|
359
|
+
const cwd = run?.cwd || queryCwd || process.cwd()
|
|
353
360
|
const sanitizeTarget = (val, def) => {
|
|
354
361
|
const clean = String(val || '').trim().replace(/[^a-zA-Z0-9_-]/g, '')
|
|
355
362
|
return clean || def
|
|
@@ -410,18 +417,10 @@ ctx.effect(() => {
|
|
|
410
417
|
const format = url.searchParams.get('format') || 'json'
|
|
411
418
|
const data = exportMoaHistory(undefined, { format, preset })
|
|
412
419
|
|
|
413
|
-
|
|
414
|
-
res.writeHead(200, {
|
|
415
|
-
'Content-Type': 'text/csv; charset=utf-8',
|
|
416
|
-
'Content-Disposition': 'attachment; filename="moa-history.csv"',
|
|
417
|
-
})
|
|
418
|
-
res.end(data)
|
|
419
|
-
return
|
|
420
|
-
}
|
|
421
|
-
|
|
420
|
+
const isCsv = format === 'csv'
|
|
422
421
|
res.writeHead(200, {
|
|
423
|
-
'Content-Type': 'application/json; charset=utf-8',
|
|
424
|
-
'Content-Disposition':
|
|
422
|
+
'Content-Type': isCsv ? 'text/csv; charset=utf-8' : 'application/json; charset=utf-8',
|
|
423
|
+
'Content-Disposition': `attachment; filename="moa-history.${isCsv ? 'csv' : 'json'}"`,
|
|
425
424
|
})
|
|
426
425
|
res.end(data)
|
|
427
426
|
} catch (err) {
|
|
@@ -559,6 +558,10 @@ ctx.effect(() => {
|
|
|
559
558
|
writeJson(res, 400, { ok: false, error: 'MoA is disabled in settings' })
|
|
560
559
|
return
|
|
561
560
|
}
|
|
561
|
+
if (payload.timeMachineUrl && !isLoopbackUrl(payload.timeMachineUrl)) {
|
|
562
|
+
writeJson(res, 400, { ok: false, error: 'Invalid timeMachineUrl: loopback host required' })
|
|
563
|
+
return
|
|
564
|
+
}
|
|
562
565
|
const presetName = payload.preset || cfg.default_preset || 'default'
|
|
563
566
|
const preset = (cfg.presets || []).find((p) => p.name === presetName) || cfg.presets?.[0]
|
|
564
567
|
|
|
@@ -568,14 +571,13 @@ ctx.effect(() => {
|
|
|
568
571
|
webServerPort: webServerPort || ctx.webServer?.port,
|
|
569
572
|
timeMachineEngine: timeMachineEngine || ctx.reflect?.get?.('timeMachineEngine', false),
|
|
570
573
|
timeMachineUrl: payload.timeMachineUrl, force: payload.force,
|
|
571
|
-
checkpoint: payload.checkpoint,
|
|
574
|
+
checkpoint: payload.checkpoint,
|
|
572
575
|
runId: payload.runId,
|
|
573
576
|
})
|
|
574
577
|
|
|
575
578
|
if (result?.skippedFiles > 0) {
|
|
576
579
|
ctx.logger?.warn?.(`[dsh-moa] Workspace scan skipped ${result.skippedFiles} file(s): ${result.skippedList.join(', ')}`)
|
|
577
580
|
}
|
|
578
|
-
|
|
579
581
|
if (result?.error) {
|
|
580
582
|
writeJson(res, 500, { ok: false, error: result.error?.message || String(result.error) })
|
|
581
583
|
return
|