@goodandready/dsh-moa 0.2.46 → 0.2.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/README.ru.md +2 -2
- package/lib/client.js +7 -1
- package/lib/index.js +1 -1
- package/lib/moa-budget.js +40 -2
- package/lib/moa-llm.js +1 -1
- package/lib/moa-multi-judge.js +64 -0
- package/lib/moa-runner.js +17 -19
- package/lib/moa-test-gate.js +55 -4
- package/lib/routes.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -150,7 +150,7 @@ Inspect line-by-line differences between candidate proposals and the curator's s
|
|
|
150
150
|
Before promoting any winning candidate files over the workspace root, `dsh-moa` invokes the local `dsh-time-machine` service to create a shadow Git checkpoint (`moa-pre-promotion: candidate-N`). If `dsh-time-machine` is absent or unreachable, file promotion is blocked (fail-closed) to protect uncheckpointed workspace files against unintended overwrites (override with `force: true` only when automated checkpoints are explicitly unneeded).
|
|
151
151
|
|
|
152
152
|
### 9. Automated Test Execution Gate
|
|
153
|
-
When `test_gate_enabled: true` and `test_command` (e.g. `npm test` or `pytest`) are configured, candidate code is executed in an ephemeral sandbox overlay. The judge receives concrete test outcomes, durations, and output logs to ground decisions in objective verification.
|
|
153
|
+
When `test_gate_enabled: true` and `test_command` (e.g. `npm test` or `pytest`) are configured, candidate code is executed in an ephemeral sandbox overlay. The test sandbox uses process containment fences and path validation to block unauthorized host file access and computed shell evasion (base64/eval/pipes), while allowing safe HTTP route literals. The judge receives concrete test outcomes, durations, and output logs to ground decisions in objective verification.
|
|
154
154
|
|
|
155
155
|
### 10. Multi-Judge Panel & Consensus Voting
|
|
156
156
|
When `multi_judge_enabled: true`, candidate solutions are independently evaluated by a panel of judge models. Winner selection supports `majority`, `highest_score`, or `unanimous` consensus strategies.
|
|
@@ -165,7 +165,7 @@ When invoking `/moa` without explicit preset flags, the router classifies prompt
|
|
|
165
165
|
As each candidate completes, live token counts and USD costs are streamed directly into the chat based on catalog pricing and vendor rates.
|
|
166
166
|
|
|
167
167
|
### 14. Cost Budget Guardrails (Trim & Abort Modes)
|
|
168
|
-
`budget_guard_enabled` and `max_budget_usd` guard against accidental spend. Mode `trim` automatically reduces the candidate pool to fit the budget, while `abort` cancels execution before token consumption.
|
|
168
|
+
`budget_guard_enabled` and `max_budget_usd` guard against accidental spend. Upfront and runtime phase guards account for candidate generation, multi-judge panel voting, and downstream synthesis prompt overhead including custom evaluation criteria (`judge_criteria`). Mode `trim` automatically reduces the candidate pool to fit the budget, while `abort` cancels execution before token consumption. In abort payloads, all panel and candidate token metrics are fully preserved.
|
|
169
169
|
|
|
170
170
|
### 15. Temperature Gradient Exploration & Per-Candidate Temperature
|
|
171
171
|
Supports individual candidate temperatures (`slot.temperature`) and `temperature_gradient_enabled` to distribute temperatures (0.2 → 0.9) across candidates for maximum architectural diversity.
|
package/README.ru.md
CHANGED
|
@@ -150,7 +150,7 @@ graph TD
|
|
|
150
150
|
Перед промоушном файлов победителя в корень рабочей области `dsh-moa` автоматически обращается к сервису `dsh-time-machine` для создания снимка (`moa-pre-promotion: candidate-N`). Если плагин недоступен или создание снимка завершилось ошибкой, перезапись файлов блокируется во избежание потери несохранённых данных (переопределяется параметром `force: true`, только если создание снимков намеренно отключено).
|
|
151
151
|
|
|
152
152
|
### 9. Автоматический шлюз выполнения тестов (Automated Test Execution Gate)
|
|
153
|
-
При включении `test_gate_enabled: true` и указании тестовой команды (`test_command`, например `npm test` или `pytest`) каждый кандидат запускается в изолированном оверлейном
|
|
153
|
+
При включении `test_gate_enabled: true` и указании тестовой команды (`test_command`, например `npm test` или `pytest`) каждый кандидат запускается в изолированном оверлейном окружении с защитой от выхода за пределы песочницы (блокировка вычисляемых shell-команд base64/eval/pipes при сохранении безопасных строковых REST-маршрутов). Судья получает объективный вердикт с кодами возврата, временем исполнения и логами тестов до вынесения итогового решения.
|
|
154
154
|
|
|
155
155
|
### 10. Коллегия судей и консенсусное голосование (Multi-Judge Panel)
|
|
156
156
|
При активации опции `multi_judge_enabled` решения кандидатов оценивает независимая панель судейских моделей с поддержкой стратегий голосования: `majority` (большинство), `highest_score` (наивысший средний балл) или `unanimous` (единогласный консенсус).
|
|
@@ -165,7 +165,7 @@ graph TD
|
|
|
165
165
|
По мере завершения каждого кандидата плагин в реальном времени транслирует в чат накопленный расход токенов и расчетную стоимость в USD на базе встроенного каталога тарифов OpenRouter и прямых цен провайдеров.
|
|
166
166
|
|
|
167
167
|
### 14. Бюджетные ограничения расходов (Cost Budget Guardrails)
|
|
168
|
-
Опция `budget_guard_enabled` с лимитом `max_budget_usd` защищает от непредвиденных
|
|
168
|
+
Опция `budget_guard_enabled` с лимитом `max_budget_usd` защищает от непредвиденных трат. Упреждающие и пофазные проверки учитывают генерацию кандидатов, панель судей и оверхед промпта синтеза с учётом пользовательских критериев (`judge_criteria`). Режим `trim` автоматически сокращает пул кандидатов до самых экономичных моделей, укладывающихся в бюджет, а режим `abort` прерывает выполнение до начала генерации. При прерывании полностью сохраняются метрики токенов кандидатов и мульти-судейской панели.
|
|
169
169
|
|
|
170
170
|
### 15. Градиент температурной диверсификации и точечная температура
|
|
171
171
|
Поддерживается как индивидуальная настройка температуры для каждого кандидата (`slot.temperature`), так и опция `temperature_gradient_enabled`, автоматически распределяющая температуры кандидатов по шкале от 0.2 до 0.9 для достижения максимального разнообразия инженерных подходов.
|
package/lib/client.js
CHANGED
|
@@ -119,6 +119,7 @@ window.__ModuleLoader__.load({
|
|
|
119
119
|
'actions.save': 'Save Configuration',
|
|
120
120
|
'actions.saving': 'Saving…',
|
|
121
121
|
'actions.saved': 'Configuration saved successfully!',
|
|
122
|
+
'actions.saved_memory_only': 'Saved to session memory only (not persisted to disk)',
|
|
122
123
|
'actions.retry': 'Reload Configuration',
|
|
123
124
|
'badge.disabled': 'MoA Disabled',
|
|
124
125
|
'config.enabled_label': 'Enable MoA (slash command and turn routing)',
|
|
@@ -297,6 +298,7 @@ window.__ModuleLoader__.load({
|
|
|
297
298
|
'actions.save': '保存配置',
|
|
298
299
|
'actions.saving': '正在保存…',
|
|
299
300
|
'actions.saved': '配置已成功保存!',
|
|
301
|
+
'actions.saved_memory_only': '已保存到会话内存(未持久化到磁盘)',
|
|
300
302
|
'actions.retry': '重新加载配置',
|
|
301
303
|
'badge.disabled': 'MoA 已禁用',
|
|
302
304
|
'config.enabled_label': '启用 MoA(斜杠命令与轮次拦截路由)',
|
|
@@ -2566,7 +2568,11 @@ window.__ModuleLoader__.load({
|
|
|
2566
2568
|
.then((r) => r.json())
|
|
2567
2569
|
.then((res) => {
|
|
2568
2570
|
if (res && res.ok) {
|
|
2569
|
-
|
|
2571
|
+
if (res.persisted === false) {
|
|
2572
|
+
setSaveStatus(t('actions.saved_memory_only'))
|
|
2573
|
+
} else {
|
|
2574
|
+
setSaveStatus(t('actions.saved'))
|
|
2575
|
+
}
|
|
2570
2576
|
if (Array.isArray(res.presets)) setPresets(res.presets)
|
|
2571
2577
|
if (res.defaultPreset) setDefaultPreset(res.defaultPreset)
|
|
2572
2578
|
setTimeout(() => setSaveStatus(''), 3000)
|
package/lib/index.js
CHANGED
|
@@ -108,7 +108,7 @@ export function apply(ctx, config) {
|
|
|
108
108
|
if (config[k] && typeof config[k] === 'object' && typeof config[k].get === 'function') continue
|
|
109
109
|
const desc = Object.getOwnPropertyDescriptor(config, k)
|
|
110
110
|
if (desc && desc.get && !desc.set) continue
|
|
111
|
-
|
|
111
|
+
bestEffort('update in-memory config key', () => { config[k] = v }, ctx.logger)
|
|
112
112
|
}
|
|
113
113
|
}
|
|
114
114
|
let liveCandidate = ctx.fiber?.config
|
package/lib/moa-budget.js
CHANGED
|
@@ -66,6 +66,35 @@ export function estimateRound2Cost(candidates = [], prices = {}, expectedOutputT
|
|
|
66
66
|
return r2Total
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
+
/**
|
|
70
|
+
* Estimates upfront token cost for multi-judge consensus panel (#158).
|
|
71
|
+
*/
|
|
72
|
+
export function estimatePanelCost(judges = [], numCandidates = 1, prices = {}, maxTokens = 2048, promptTokens = 500) {
|
|
73
|
+
if (!Array.isArray(judges) || judges.length === 0 || numCandidates <= 1) return 0
|
|
74
|
+
let totalCost = 0
|
|
75
|
+
const inputTokens = promptTokens + (maxTokens * numCandidates) + 150
|
|
76
|
+
const outputTokens = Math.min(maxTokens, 1024)
|
|
77
|
+
for (const j of judges) {
|
|
78
|
+
if (j?.provider && j?.model) {
|
|
79
|
+
const cost = estimateTokenCost(j, { inputTokens, outputTokens }, prices)?.costUsd || 0
|
|
80
|
+
totalCost += cost
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return totalCost
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Calculates accurate synthesis prompt tokens accounting for full user prompt,
|
|
88
|
+
* judge criteria, and reference candidate texts (#158).
|
|
89
|
+
*/
|
|
90
|
+
export function calcSynthesisPromptTokens(userPrompt = '', referenceOutputs = [], judgeCriteria = '', overhead = 2600) {
|
|
91
|
+
const promptChars = (typeof userPrompt === 'string' ? userPrompt.length : 0) +
|
|
92
|
+
(typeof judgeCriteria === 'string' ? judgeCriteria.length : 0) +
|
|
93
|
+
(Array.isArray(referenceOutputs) ? referenceOutputs.reduce((acc, r) => acc + (r.text?.length || 0), 0) : 0) +
|
|
94
|
+
overhead
|
|
95
|
+
return Math.max(50, Math.ceil(promptChars / 4))
|
|
96
|
+
}
|
|
97
|
+
|
|
69
98
|
/**
|
|
70
99
|
* Phase-by-phase runtime budget check (#158).
|
|
71
100
|
* Checks if current spent + next phase estimate would exceed budget.
|
|
@@ -108,6 +137,7 @@ export function applyBudgetGuardrails({
|
|
|
108
137
|
promptLength = 1000,
|
|
109
138
|
projectContext = '',
|
|
110
139
|
messages = [],
|
|
140
|
+
judgeCriteria = '',
|
|
111
141
|
maxTokens = 2048,
|
|
112
142
|
peerCritiqueEnabled = false,
|
|
113
143
|
aggregator = null,
|
|
@@ -122,8 +152,11 @@ export function applyBudgetGuardrails({
|
|
|
122
152
|
}
|
|
123
153
|
}
|
|
124
154
|
|
|
125
|
-
// Calculate full input tokens from userPrompt length + messages context + system prompt overhead
|
|
155
|
+
// Calculate full input tokens from userPrompt length + judgeCriteria + messages context + system prompt overhead
|
|
126
156
|
let contextChars = Number(promptLength) || 1000
|
|
157
|
+
if (typeof judgeCriteria === 'string' && judgeCriteria.length > 0 && !String(promptLength).includes(String(judgeCriteria.length))) {
|
|
158
|
+
contextChars += judgeCriteria.length
|
|
159
|
+
}
|
|
127
160
|
if (typeof projectContext === 'string' && projectContext.length > 0 && !String(promptLength).includes(String(projectContext.length))) {
|
|
128
161
|
contextChars += projectContext.length
|
|
129
162
|
}
|
|
@@ -245,8 +278,11 @@ export function createBudgetAbortPayload({
|
|
|
245
278
|
isFastMode = false,
|
|
246
279
|
collectedCtx = {},
|
|
247
280
|
startTime = Date.now(),
|
|
281
|
+
multiJudgeResult = null,
|
|
248
282
|
}) {
|
|
249
283
|
const aggLabel = (primaryJudge?.provider && primaryJudge?.model) ? `${primaryJudge.provider}:${primaryJudge.model}` : "none";
|
|
284
|
+
const refTokens = referenceOutputs.reduce((acc, r) => acc + (r.usage?.totalTokens || 0), 0);
|
|
285
|
+
const panelTokens = multiJudgeResult?.totalUsage?.totalTokens || 0;
|
|
250
286
|
return {
|
|
251
287
|
kind: "failure",
|
|
252
288
|
content: `🛑 Budget Guardrail Abort: ${reason}`,
|
|
@@ -259,11 +295,13 @@ export function createBudgetAbortPayload({
|
|
|
259
295
|
winnerModel: "none",
|
|
260
296
|
promotedFiles: [],
|
|
261
297
|
usage: {
|
|
262
|
-
totalTokens:
|
|
298
|
+
totalTokens: refTokens + panelTokens,
|
|
263
299
|
totalCostUsd: Number(currentSpentUsd.toFixed(5)),
|
|
264
300
|
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
265
301
|
aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 },
|
|
302
|
+
...(multiJudgeResult ? { multiJudge: multiJudgeResult.totalUsage || { totalTokens: panelTokens } } : {}),
|
|
266
303
|
},
|
|
304
|
+
multiJudgeResult: multiJudgeResult || undefined,
|
|
267
305
|
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
268
306
|
skippedList: collectedCtx?.skippedList || [],
|
|
269
307
|
durationMs: Date.now() - startTime,
|
package/lib/moa-llm.js
CHANGED
|
@@ -26,7 +26,7 @@ function combineSignals(signal, timeoutSignal) {
|
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
28
|
export function createLlmCaller(ctx) {
|
|
29
|
-
return async ({ provider, model, messages, temperature = 0.6, maxTokens =
|
|
29
|
+
return async ({ provider, model, messages, temperature = 0.6, maxTokens = 2048, timeoutMs, onStreamDelta, signal, stop, reasoningEffort }) => {
|
|
30
30
|
if (!ctx.llm) {
|
|
31
31
|
throw new Error('ctx.llm is not available in cordis context')
|
|
32
32
|
}
|
package/lib/moa-multi-judge.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { slotLabel } from './moa-prompts.js'
|
|
7
7
|
import { estimateTokenCost } from './pricing.js'
|
|
8
|
+
import { estimatePanelCost, estimateDownstreamCost, checkPhaseBudget, calcSynthesisPromptTokens } from './moa-budget.js'
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* Extracts code fences and block-level assets from markdown text.
|
|
@@ -160,6 +161,7 @@ export async function executeMultiJudgePanel({
|
|
|
160
161
|
onProgress,
|
|
161
162
|
prices = {},
|
|
162
163
|
blind = false,
|
|
164
|
+
maxTokens = 2048,
|
|
163
165
|
}) {
|
|
164
166
|
const activeCandidates = (candidates || []).filter((c) => c && c.ok)
|
|
165
167
|
if (activeCandidates.length === 0) {
|
|
@@ -240,6 +242,7 @@ CONSENSUS_REASONING: [one concise sentence explaining your pick]
|
|
|
240
242
|
{ role: 'system', content: 'You are an objective AI code evaluation judge.' },
|
|
241
243
|
{ role: 'user', content: judgePrompt },
|
|
242
244
|
],
|
|
245
|
+
maxTokens,
|
|
243
246
|
signal: abortCtrl.signal,
|
|
244
247
|
})
|
|
245
248
|
|
|
@@ -392,3 +395,64 @@ CONSENSUS_REASONING: [one concise sentence explaining your pick]
|
|
|
392
395
|
consensusReport,
|
|
393
396
|
}
|
|
394
397
|
}
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* Runs the Multi-Judge consensus phase with budget pre-checking (#158, #168).
|
|
402
|
+
*/
|
|
403
|
+
export async function runMultiJudgePhase({
|
|
404
|
+
preset = {},
|
|
405
|
+
successfulRefs = [],
|
|
406
|
+
primaryJudge,
|
|
407
|
+
userPrompt = '',
|
|
408
|
+
judgeCriteria = '',
|
|
409
|
+
prices = {},
|
|
410
|
+
maxTokens = 2048,
|
|
411
|
+
callLlm,
|
|
412
|
+
signal,
|
|
413
|
+
onProgress,
|
|
414
|
+
currentSpent = 0,
|
|
415
|
+
}) {
|
|
416
|
+
if (!preset?.multi_judge_enabled || successfulRefs.length <= 1) {
|
|
417
|
+
return { skipped: true, result: null }
|
|
418
|
+
}
|
|
419
|
+
const panelJudges = (preset.judge_models && preset.judge_models.length > 0) ? preset.judge_models : [primaryJudge]
|
|
420
|
+
const panelMaxTokens = Math.min(typeof preset.max_tokens === 'number' && preset.max_tokens > 0 ? preset.max_tokens : 2048, 2048)
|
|
421
|
+
|
|
422
|
+
if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0) {
|
|
423
|
+
const promptTokens = calcSynthesisPromptTokens(userPrompt, successfulRefs, judgeCriteria)
|
|
424
|
+
const panelEst = estimatePanelCost(panelJudges, successfulRefs.length, prices, panelMaxTokens, promptTokens)
|
|
425
|
+
const downEst = estimateDownstreamCost(primaryJudge, [], successfulRefs.length, prices, maxTokens, promptTokens)
|
|
426
|
+
const phaseCheck = checkPhaseBudget({
|
|
427
|
+
phaseName: 'Multi-Judge Panel',
|
|
428
|
+
currentSpentUsd: currentSpent,
|
|
429
|
+
nextPhaseCostUsd: panelEst + downEst,
|
|
430
|
+
maxBudgetUsd: preset.max_budget_usd,
|
|
431
|
+
action: preset.budget_action || 'trim',
|
|
432
|
+
enabled: true,
|
|
433
|
+
})
|
|
434
|
+
if (!phaseCheck.allowed) {
|
|
435
|
+
if (preset.budget_action === 'abort') {
|
|
436
|
+
return { abort: true, reason: phaseCheck.reason }
|
|
437
|
+
}
|
|
438
|
+
if (typeof onProgress === 'function') {
|
|
439
|
+
onProgress(`✂️ *Skipping Multi-Judge Panel to stay within budget ($${preset.max_budget_usd.toFixed(4)})*\n`)
|
|
440
|
+
}
|
|
441
|
+
return { skipped: true, result: null }
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
const result = await executeMultiJudgePanel({
|
|
446
|
+
judges: panelJudges,
|
|
447
|
+
candidates: successfulRefs,
|
|
448
|
+
userPrompt,
|
|
449
|
+
callLlm,
|
|
450
|
+
strategy: preset.judge_voting_strategy || 'majority',
|
|
451
|
+
prices: prices || {},
|
|
452
|
+
blind: Boolean(preset.blind_evaluation),
|
|
453
|
+
maxTokens: panelMaxTokens,
|
|
454
|
+
signal,
|
|
455
|
+
onProgress,
|
|
456
|
+
})
|
|
457
|
+
return { skipped: false, result }
|
|
458
|
+
}
|
package/lib/moa-runner.js
CHANGED
|
@@ -8,8 +8,8 @@ import { slotLabel, isBroadPromptRequiringQuestions, buildQuestionSynthesisPromp
|
|
|
8
8
|
import { parseWinnerIndex, parseRecommendedAssembler } from './moa-parser.js'
|
|
9
9
|
import { REFERENCE_SYSTEM_PROMPT, callWithTransientRetry, runReferencesParallel } from './moa-candidates.js'
|
|
10
10
|
import { executeTestGateForCandidates } from './moa-test-gate.js'
|
|
11
|
-
import { executeMultiJudgePanel, buildCompositeBlockDirectives } from './moa-multi-judge.js'
|
|
12
|
-
import { applyBudgetGuardrails, checkPhaseBudget, estimateRound2Cost, estimateDownstreamCost, createBudgetAbortPayload } from './moa-budget.js'
|
|
11
|
+
import { executeMultiJudgePanel, buildCompositeBlockDirectives, runMultiJudgePhase } from './moa-multi-judge.js'
|
|
12
|
+
import { applyBudgetGuardrails, checkPhaseBudget, estimateRound2Cost, estimateDownstreamCost, calcSynthesisPromptTokens, createBudgetAbortPayload } from './moa-budget.js'
|
|
13
13
|
import { extractPriorTurnBaseline, pruneMultiTurnMessages } from './moa-context.js'
|
|
14
14
|
import { generateMoABenchmarkReport } from './moa-report.js'
|
|
15
15
|
// Re-exports for consumers & backward compatibility
|
|
@@ -80,7 +80,8 @@ export async function runMoAPipeline({
|
|
|
80
80
|
maxBudgetUsd: preset?.max_budget_usd,
|
|
81
81
|
action: preset?.budget_action || 'trim',
|
|
82
82
|
prices,
|
|
83
|
-
promptLength: (userPrompt?.length || 1000) + (projectContext ? projectContext.length : 0),
|
|
83
|
+
promptLength: (userPrompt?.length || 1000) + (judgeCriteria?.length || 0) + (projectContext ? projectContext.length : 0),
|
|
84
|
+
judgeCriteria,
|
|
84
85
|
projectContext,
|
|
85
86
|
messages,
|
|
86
87
|
maxTokens,
|
|
@@ -384,19 +385,15 @@ export async function runMoAPipeline({
|
|
|
384
385
|
return { error: new Error('Turn aborted') }
|
|
385
386
|
}
|
|
386
387
|
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
prices: prices || {},
|
|
395
|
-
blind: Boolean(preset.blind_evaluation),
|
|
396
|
-
signal,
|
|
397
|
-
onProgress,
|
|
398
|
-
})
|
|
388
|
+
const currentRefSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0)
|
|
389
|
+
const panelPhase = await runMultiJudgePhase({
|
|
390
|
+
preset, successfulRefs, primaryJudge, userPrompt, judgeCriteria, prices, maxTokens, callLlm, signal, onProgress, currentSpent: currentRefSpent,
|
|
391
|
+
})
|
|
392
|
+
if (panelPhase.abort) {
|
|
393
|
+
await cleanMoaWorkspaces(cwd, { runId })
|
|
394
|
+
return createBudgetAbortPayload({ reason: panelPhase.reason, primaryJudge, referenceOutputs, currentSpentUsd: currentRefSpent, preset, isRefinement, isFastMode, collectedCtx, startTime })
|
|
399
395
|
}
|
|
396
|
+
multiJudgeResult = panelPhase.result
|
|
400
397
|
|
|
401
398
|
// 7d. Feature 3: Composite Block Merge Directive
|
|
402
399
|
const compositeMergeDirective = (preset?.composite_merge_enabled && successfulRefs.length > 1)
|
|
@@ -404,13 +401,14 @@ export async function runMoAPipeline({
|
|
|
404
401
|
: null
|
|
405
402
|
|
|
406
403
|
// 8. Synthesis phase via primary judge or fallback chain
|
|
404
|
+
const synthPromptTokens = calcSynthesisPromptTokens(userPrompt, successfulRefs, judgeCriteria)
|
|
407
405
|
if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode) {
|
|
408
406
|
const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0) + (multiJudgeResult?.totalCostUsd || 0)
|
|
409
|
-
const downNext = estimateDownstreamCost(primaryJudge, [], successfulRefs.length, prices, maxTokens)
|
|
407
|
+
const downNext = estimateDownstreamCost(primaryJudge, [], successfulRefs.length, prices, maxTokens, synthPromptTokens)
|
|
410
408
|
const phaseCheck = checkPhaseBudget({ phaseName: 'Judge Synthesis', currentSpentUsd: currentSpent, nextPhaseCostUsd: downNext, maxBudgetUsd: preset.max_budget_usd, action: preset.budget_action || 'trim', enabled: true })
|
|
411
409
|
if (!phaseCheck.allowed && fallbackJudges.length === 0) {
|
|
412
410
|
await cleanMoaWorkspaces(cwd, { runId })
|
|
413
|
-
return createBudgetAbortPayload({ reason: phaseCheck.reason, primaryJudge, referenceOutputs, currentSpentUsd: currentSpent, preset, isRefinement, isFastMode, collectedCtx, startTime })
|
|
411
|
+
return createBudgetAbortPayload({ reason: phaseCheck.reason, primaryJudge, referenceOutputs, currentSpentUsd: currentSpent, preset, isRefinement, isFastMode, collectedCtx, startTime, multiJudgeResult })
|
|
414
412
|
}
|
|
415
413
|
}
|
|
416
414
|
const synthesisPrompt = buildSynthesisPrompt(userPrompt, referenceOutputs, judgeCriteria, {
|
|
@@ -432,7 +430,7 @@ export async function runMoAPipeline({
|
|
|
432
430
|
|
|
433
431
|
if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode) {
|
|
434
432
|
const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0) + (multiJudgeResult?.totalCostUsd || 0)
|
|
435
|
-
const judgeCost = estimateDownstreamCost(currentJudge, [], successfulRefs.length, prices, maxTokens)
|
|
433
|
+
const judgeCost = estimateDownstreamCost(currentJudge, [], successfulRefs.length, prices, maxTokens, synthPromptTokens)
|
|
436
434
|
const phaseCheck = checkPhaseBudget({ phaseName: `Judge Synthesis (${currentLabel})`, currentSpentUsd: currentSpent, nextPhaseCostUsd: judgeCost, maxBudgetUsd: preset.max_budget_usd, action: preset.budget_action || 'trim', enabled: true })
|
|
437
435
|
if (!phaseCheck.allowed) {
|
|
438
436
|
lastJudgeError = new Error(phaseCheck.reason)
|
|
@@ -489,7 +487,7 @@ export async function runMoAPipeline({
|
|
|
489
487
|
if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode && (lastJudgeError?.message?.includes('exceeds max budget') || lastJudgeError?.message?.includes('Budget Guardrail') || preset?.budget_action === 'abort')) {
|
|
490
488
|
await cleanMoaWorkspaces(cwd, { runId })
|
|
491
489
|
const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0) + (multiJudgeResult?.totalCostUsd || 0)
|
|
492
|
-
return createBudgetAbortPayload({ reason: lastJudgeError?.message || 'Budget exceeded before judge synthesis', primaryJudge, referenceOutputs, currentSpentUsd: currentSpent, preset, isRefinement, isFastMode, collectedCtx, startTime })
|
|
490
|
+
return createBudgetAbortPayload({ reason: lastJudgeError?.message || 'Budget exceeded before judge synthesis', primaryJudge, referenceOutputs, currentSpentUsd: currentSpent, preset, isRefinement, isFastMode, collectedCtx, startTime, multiJudgeResult })
|
|
493
491
|
}
|
|
494
492
|
const firstErr = lastJudgeError ? (lastJudgeError.message || String(lastJudgeError)) : 'unknown error'
|
|
495
493
|
synthesizedText = `⚠️ *[Aggregator error: ${firstErr}. Fallback candidate outputs:]*\n\n` +
|
package/lib/moa-test-gate.js
CHANGED
|
@@ -43,7 +43,9 @@ export function resolveCandidateDir(baseDir, candidateIndex, options = {}) {
|
|
|
43
43
|
}
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
-
const
|
|
46
|
+
const COMPUTED_SHELL_EVASION = /(?:base64\s+(?:-d|--decode|-D)|\beval\s+["\']?\$|\bxxd\s+-r|\bprintf\s+[^|\n]+\|\s*(?:ba|z|da)?sh|\bopenssl\s+enc\s+-d)/i
|
|
47
|
+
|
|
48
|
+
const UNIX_ROOT_PREFIX = /^\/(?:home|root|etc|var|tmp|opt|mnt|srv|media|proc|sys|dev|usr|bin|lib|lib64|run\/(?:user|systemd|lock|credentials|secrets|udev|initramfs|mount|shm|dbus|sshd|motd)|sbin|boot|Users\/(?:[a-zA-Z0-9_\-.]+)|Applications|Library|System|Volumes|private)(?:\/|$)/
|
|
47
49
|
|
|
48
50
|
/**
|
|
49
51
|
* Validates that an executed script or command does not reference files outside allowed roots (#161, #225).
|
|
@@ -189,6 +191,18 @@ export async function runCandidateTestGate({
|
|
|
189
191
|
if (fsSync.existsSync(scriptPath) && !fsSync.statSync(scriptPath).isDirectory()) {
|
|
190
192
|
try {
|
|
191
193
|
const scriptText = fsSync.readFileSync(scriptPath, 'utf8')
|
|
194
|
+
if (COMPUTED_SHELL_EVASION.test(scriptText)) {
|
|
195
|
+
const durationMs = Date.now() - startTime
|
|
196
|
+
return {
|
|
197
|
+
candidateIndex,
|
|
198
|
+
passed: false,
|
|
199
|
+
exitCode: 1,
|
|
200
|
+
durationMs,
|
|
201
|
+
command: resolvedCmd,
|
|
202
|
+
output: 'Access denied: test script contains obfuscated or computed command execution',
|
|
203
|
+
summary: `FAIL (code 1, ${durationMs}ms)`,
|
|
204
|
+
}
|
|
205
|
+
}
|
|
192
206
|
const leaked = scanPathTokensForLeak(scriptText, allowedRoots, executionDir)
|
|
193
207
|
if (leaked) {
|
|
194
208
|
const durationMs = Date.now() - startTime
|
|
@@ -202,7 +216,9 @@ export async function runCandidateTestGate({
|
|
|
202
216
|
summary: `FAIL (code 1, ${durationMs}ms)`,
|
|
203
217
|
}
|
|
204
218
|
}
|
|
205
|
-
} catch {
|
|
219
|
+
} catch (scanErr) {
|
|
220
|
+
logger.debug('[dsh-moa:testGate] candidate script read error:', scanErr?.message || scanErr)
|
|
221
|
+
}
|
|
206
222
|
}
|
|
207
223
|
}
|
|
208
224
|
|
|
@@ -212,6 +228,18 @@ export async function runCandidateTestGate({
|
|
|
212
228
|
try {
|
|
213
229
|
const pkgObj = JSON.parse(fsSync.readFileSync(candPkgFile, 'utf8'))
|
|
214
230
|
const scriptsContent = Object.values(pkgObj.scripts || {}).join('\n')
|
|
231
|
+
if (COMPUTED_SHELL_EVASION.test(scriptsContent)) {
|
|
232
|
+
const durationMs = Date.now() - startTime
|
|
233
|
+
return {
|
|
234
|
+
candidateIndex,
|
|
235
|
+
passed: false,
|
|
236
|
+
exitCode: 1,
|
|
237
|
+
durationMs,
|
|
238
|
+
command: resolvedCmd,
|
|
239
|
+
output: 'Access denied: package.json test script contains obfuscated or computed command execution',
|
|
240
|
+
summary: `FAIL (code 1, ${durationMs}ms)`,
|
|
241
|
+
}
|
|
242
|
+
}
|
|
215
243
|
const leaked = scanPathTokensForLeak(scriptsContent, allowedRoots, executionDir)
|
|
216
244
|
if (leaked) {
|
|
217
245
|
const durationMs = Date.now() - startTime
|
|
@@ -225,7 +253,9 @@ export async function runCandidateTestGate({
|
|
|
225
253
|
summary: `FAIL (code 1, ${durationMs}ms)`,
|
|
226
254
|
}
|
|
227
255
|
}
|
|
228
|
-
} catch {
|
|
256
|
+
} catch (pkgErr) {
|
|
257
|
+
logger.debug('[dsh-moa:testGate] candidate package.json read error:', pkgErr?.message || pkgErr)
|
|
258
|
+
}
|
|
229
259
|
}
|
|
230
260
|
|
|
231
261
|
// Setup isolated HOME & node roots
|
|
@@ -245,6 +275,8 @@ export async function runCandidateTestGate({
|
|
|
245
275
|
...(candidateDir && candidateDir !== executionDir ? [candidateDir] : []),
|
|
246
276
|
nodeRoot,
|
|
247
277
|
nodeBinDir,
|
|
278
|
+
...(process.env.NVM_DIR ? [process.env.NVM_DIR] : []),
|
|
279
|
+
...(nodeRoot.includes("/.nvm/") ? [path.resolve(nodeRoot, "../..")] : []),
|
|
248
280
|
...(process.platform === 'win32'
|
|
249
281
|
? [process.env.SYSTEMROOT || 'C:\\Windows']
|
|
250
282
|
: ['/usr', '/lib', '/lib64', '/etc']),
|
|
@@ -256,7 +288,8 @@ export async function runCandidateTestGate({
|
|
|
256
288
|
const cp = require('node:child_process');
|
|
257
289
|
const path = require('node:path');
|
|
258
290
|
const allowed = ${JSON.stringify(allowedReads)};
|
|
259
|
-
const
|
|
291
|
+
const COMPUTED_SHELL_EVASION = /(?:base64\\s+(?:-d|--decode|-D)|\\beval\\s+["\']?\\$|\\bxxd\\s+-r|\\bprintf\\s+[^|\\n]+\\|\\s*(?:ba|z|da)?sh|\\bopenssl\\s+enc\\s+-d)/i;
|
|
292
|
+
const UNIX_ROOT_PREFIX = /^\\/(?:home|root|etc|var|tmp|opt|mnt|srv|media|proc|sys|dev|usr|bin|lib|lib64|run\\/(?:user|systemd|lock|credentials|secrets|udev|initramfs|mount|shm|dbus|sshd|motd)|sbin|boot|Users\\/(?:[a-zA-Z0-9_\\-.]+)|Applications|Library|System|Volumes|private)(?:\\/|$)/;
|
|
260
293
|
function checkSinglePath(p) {
|
|
261
294
|
if (!p || typeof p !== 'string') return true;
|
|
262
295
|
if (/^\\/(usr\\/(bin|lib|include)|bin|lib|lib64|etc|dev)(\\/|$)/.test(p) && !/\\.(txt|json|env|npmrc|key|ssh|md)/i.test(p)) return true;
|
|
@@ -277,6 +310,10 @@ function scanStringForPaths(str) {
|
|
|
277
310
|
return true;
|
|
278
311
|
}
|
|
279
312
|
function guardArgs(cmd, args) {
|
|
313
|
+
const full = [cmd, ...(Array.isArray(args) ? args : [])].join(' ');
|
|
314
|
+
if (COMPUTED_SHELL_EVASION.test(full)) {
|
|
315
|
+
throw new Error('ERR_ACCESS_DENIED: child process computed execution pattern blocked: ' + full);
|
|
316
|
+
}
|
|
280
317
|
if (cmd && !scanStringForPaths(String(cmd))) {
|
|
281
318
|
throw new Error('ERR_ACCESS_DENIED: child process command outside sandbox: ' + cmd);
|
|
282
319
|
}
|
|
@@ -318,6 +355,20 @@ cp.execSync = function(cmd, opts) {
|
|
|
318
355
|
guardArgs(cmd, [cmd]);
|
|
319
356
|
return origExecSync.apply(this, arguments);
|
|
320
357
|
};
|
|
358
|
+
const fsMod = require('node:fs');
|
|
359
|
+
function guardFs(p, isWrite) {
|
|
360
|
+
if (!p || typeof p !== 'string') return;
|
|
361
|
+
const isAbs = path.isAbsolute(p);
|
|
362
|
+
const isUp = p.startsWith('..');
|
|
363
|
+
if (!isAbs && !isUp) return;
|
|
364
|
+
const resolved = path.resolve(process.cwd(), p);
|
|
365
|
+
const checkRoots = isWrite ? [process.cwd(), process.env.HOME].filter(Boolean) : allowed;
|
|
366
|
+
const ok = checkRoots.some(r => r && (resolved === r || resolved.startsWith(r + path.sep)));
|
|
367
|
+
if (!ok) throw new Error('ERR_ACCESS_DENIED: fs access outside sandbox: ' + p);
|
|
368
|
+
}
|
|
369
|
+
const origRF = fsMod.readFileSync; fsMod.readFileSync = function(p) { guardFs(p, false); return origRF.apply(this, arguments); };
|
|
370
|
+
const origWF = fsMod.writeFileSync; fsMod.writeFileSync = function(p) { guardFs(p, true); return origWF.apply(this, arguments); };
|
|
371
|
+
const origAF = fsMod.appendFileSync; fsMod.appendFileSync = function(p) { guardFs(p, true); return origAF.apply(this, arguments); };
|
|
321
372
|
`
|
|
322
373
|
fsSync.writeFileSync(fenceFile, fenceCode, 'utf8')
|
|
323
374
|
|
package/lib/routes.js
CHANGED
|
@@ -303,7 +303,7 @@ ctx.effect(() => {
|
|
|
303
303
|
}
|
|
304
304
|
const effectiveCwd = recordedRun.cwd
|
|
305
305
|
const promotedFiles = await promoteCandidateWorkspace(effectiveCwd, candidateIndex, {
|
|
306
|
-
runId:
|
|
306
|
+
runId: recordedRun.id,
|
|
307
307
|
keepMoa: true,
|
|
308
308
|
port: ctx.webServer?.port,
|
|
309
309
|
timeMachineEngine: timeMachineEngine || ctx.reflect?.get?.('timeMachineEngine', false),
|