@goodandready/dsh-moa 0.2.44 → 0.2.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/file-workspace.js +35 -27
- package/lib/history.js +1 -0
- package/lib/index.js +12 -1
- package/lib/moa-budget.js +134 -30
- package/lib/moa-runner.js +99 -106
- package/lib/moa-test-gate.js +210 -33
- package/lib/routes.js +80 -110
- package/package.json +1 -1
package/lib/moa-runner.js
CHANGED
|
@@ -9,7 +9,7 @@ import { parseWinnerIndex, parseRecommendedAssembler } from './moa-parser.js'
|
|
|
9
9
|
import { REFERENCE_SYSTEM_PROMPT, callWithTransientRetry, runReferencesParallel } from './moa-candidates.js'
|
|
10
10
|
import { executeTestGateForCandidates } from './moa-test-gate.js'
|
|
11
11
|
import { executeMultiJudgePanel, buildCompositeBlockDirectives } from './moa-multi-judge.js'
|
|
12
|
-
import { applyBudgetGuardrails } from './moa-budget.js'
|
|
12
|
+
import { applyBudgetGuardrails, checkPhaseBudget, estimateRound2Cost, estimateDownstreamCost, createBudgetAbortPayload } from './moa-budget.js'
|
|
13
13
|
import { extractPriorTurnBaseline, pruneMultiTurnMessages } from './moa-context.js'
|
|
14
14
|
import { generateMoABenchmarkReport } from './moa-report.js'
|
|
15
15
|
// Re-exports for consumers & backward compatibility
|
|
@@ -27,13 +27,8 @@ export async function runMoAPipeline({
|
|
|
27
27
|
messages = [],
|
|
28
28
|
preset,
|
|
29
29
|
callLlm,
|
|
30
|
-
cwd = process.cwd(),
|
|
31
|
-
|
|
32
|
-
onProgress = null,
|
|
33
|
-
historyFilePath = null,
|
|
34
|
-
liveCanvas = null,
|
|
35
|
-
onStreamDelta = null,
|
|
36
|
-
signal = null,
|
|
30
|
+
cwd = process.cwd(), prices = {}, onProgress = null,
|
|
31
|
+
historyFilePath = null, liveCanvas = null, onStreamDelta = null, signal = null,
|
|
37
32
|
testGateEnabled = null, testCommand = null, execFn = null,
|
|
38
33
|
webServerPort = null, timeMachineEngine = null, timeMachineUrl = null,
|
|
39
34
|
force = isTestEnvironment() ? true : false, checkpoint = true, checkpointFn = null, runId: customRunId = null,
|
|
@@ -94,14 +89,7 @@ export async function runMoAPipeline({
|
|
|
94
89
|
judgeModels: preset?.multi_judge_enabled ? preset?.judge_models : [],
|
|
95
90
|
})
|
|
96
91
|
if (!budgetCheck.allowed) {
|
|
97
|
-
return {
|
|
98
|
-
aggregator: slotLabel(primaryJudge), references: [], presetName: preset?.name || 'default',
|
|
99
|
-
isRefinement, isFastMode, winningIndex: 0, winnerModel: 'none', promotedFiles: [],
|
|
100
|
-
usage: { totalTokens: 0, totalCostUsd: 0, candidates: [], aggregator: { totalTokens: 0, costUsd: 0 } },
|
|
101
|
-
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
102
|
-
skippedList: collectedCtx?.skippedList || [],
|
|
103
|
-
durationMs: Date.now() - startTime,
|
|
104
|
-
}
|
|
92
|
+
return createBudgetAbortPayload({ reason: budgetCheck.reason, primaryJudge, referenceOutputs: [], currentSpentUsd: 0, preset, isRefinement, isFastMode, collectedCtx, startTime })
|
|
105
93
|
}
|
|
106
94
|
const effectiveReferences = budgetCheck.references
|
|
107
95
|
if (budgetCheck.action === 'trim' && typeof onProgress === 'function') {
|
|
@@ -145,26 +133,14 @@ export async function runMoAPipeline({
|
|
|
145
133
|
const successfulRefs = referenceOutputs.filter((r) => r.ok)
|
|
146
134
|
if (successfulRefs.length === 0) {
|
|
147
135
|
const reasons = referenceOutputs.map((r) => `${r.label}: ${r.error || 'unknown error'}`).join('; ')
|
|
136
|
+
const totalTokens = referenceOutputs.reduce((acc, r) => acc + (r.usage?.totalTokens || 0), 0)
|
|
137
|
+
const totalCostUsd = Number((referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0)).toFixed(5))
|
|
148
138
|
return {
|
|
149
|
-
kind: 'failure',
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
isRefinement,
|
|
155
|
-
isFastMode,
|
|
156
|
-
winningIndex: 0,
|
|
157
|
-
winnerModel: 'none',
|
|
158
|
-
promotedFiles: [],
|
|
159
|
-
usage: {
|
|
160
|
-
totalTokens: 0,
|
|
161
|
-
totalCostUsd: 0,
|
|
162
|
-
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
163
|
-
aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 },
|
|
164
|
-
},
|
|
165
|
-
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
166
|
-
skippedList: collectedCtx?.skippedList || [],
|
|
167
|
-
durationMs: Date.now() - startTime,
|
|
139
|
+
kind: 'failure', content: `⚠️ All advisor models (${referenceOutputs.length}) failed: ${reasons}`,
|
|
140
|
+
aggregator: slotLabel(primaryJudge), references: referenceOutputs, presetName: preset?.name || 'default',
|
|
141
|
+
isRefinement, isFastMode, winningIndex: 0, winnerModel: 'none', promotedFiles: [],
|
|
142
|
+
usage: { totalTokens, totalCostUsd, candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })), aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 } },
|
|
143
|
+
skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs: Date.now() - startTime,
|
|
168
144
|
}
|
|
169
145
|
}
|
|
170
146
|
// 5. Extract file blocks & write candidate workspaces
|
|
@@ -240,25 +216,11 @@ export async function runMoAPipeline({
|
|
|
240
216
|
logger.warn('[dsh-moa] Failed to record questionnaire run in history:', histErr?.message || histErr)
|
|
241
217
|
}
|
|
242
218
|
return {
|
|
243
|
-
kind: 'questions',
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
isRefinement,
|
|
249
|
-
isFastMode,
|
|
250
|
-
winningIndex: 0,
|
|
251
|
-
winnerModel: 'none',
|
|
252
|
-
promotedFiles: [],
|
|
253
|
-
usage: {
|
|
254
|
-
totalTokens,
|
|
255
|
-
totalCostUsd,
|
|
256
|
-
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
257
|
-
aggregator: qUsage,
|
|
258
|
-
},
|
|
259
|
-
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
260
|
-
skippedList: collectedCtx?.skippedList || [],
|
|
261
|
-
durationMs,
|
|
219
|
+
kind: 'questions', content: questionsContent, aggregator: slotLabel(primaryJudge),
|
|
220
|
+
references: referenceOutputs, presetName: preset?.name || 'default', isRefinement, isFastMode,
|
|
221
|
+
winningIndex: 0, winnerModel: 'none', promotedFiles: [],
|
|
222
|
+
usage: { totalTokens, totalCostUsd, candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })), aggregator: qUsage },
|
|
223
|
+
skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs,
|
|
262
224
|
}
|
|
263
225
|
}
|
|
264
226
|
|
|
@@ -282,15 +244,34 @@ export async function runMoAPipeline({
|
|
|
282
244
|
if (signal?.aborted || promoteErr?.message?.includes('aborted') || promoteErr?.name === 'AbortError' || promoteErr?.message === 'Turn aborted') {
|
|
283
245
|
return { error: new Error('Turn aborted') }
|
|
284
246
|
}
|
|
285
|
-
|
|
247
|
+
const totalTokens = referenceOutputs.reduce((acc, r) => acc + (r.usage?.totalTokens || 0), 0)
|
|
248
|
+
const totalCostUsd = Number((referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0)).toFixed(5))
|
|
249
|
+
const durationMs = Date.now() - startTime
|
|
250
|
+
try {
|
|
251
|
+
await recordMoaRunAsync({
|
|
252
|
+
id: runId, cwd, prompt: userPrompt, preset: preset?.name || 'default',
|
|
253
|
+
isRefinement, isFastMode: true, candidates: candidatesForHistory(referenceOutputs),
|
|
254
|
+
aggregator: null, winnerIndex: 0, winnerModel: 'none',
|
|
255
|
+
promotedFiles: [], totalTokens, totalCostUsd, durationMs,
|
|
256
|
+
error: promoteErr?.message || String(promoteErr),
|
|
257
|
+
}, historyFilePath || undefined)
|
|
258
|
+
} catch (histErr) {
|
|
259
|
+
logger.warn('[dsh-moa] Failed to record fast synthesis failure in history:', histErr?.message || histErr)
|
|
260
|
+
}
|
|
261
|
+
return {
|
|
262
|
+
kind: 'failure', error: promoteErr,
|
|
286
263
|
content: '🛑 Checkpoint or Promotion Failure: ' + (promoteErr?.message || String(promoteErr)),
|
|
287
264
|
aggregator: 'Fast Mode (Direct)', references: referenceOutputs,
|
|
288
265
|
presetName: preset?.name || 'default', isRefinement, isFastMode: true,
|
|
289
|
-
winningIndex:
|
|
290
|
-
usage: {
|
|
266
|
+
winningIndex: 0, winnerModel: 'none', promotedFiles: [], runId,
|
|
267
|
+
usage: {
|
|
268
|
+
totalTokens, totalCostUsd,
|
|
269
|
+
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
270
|
+
aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 },
|
|
271
|
+
},
|
|
291
272
|
skippedFiles: collectedCtx?.skippedFiles || 0,
|
|
292
273
|
skippedList: collectedCtx?.skippedList || [],
|
|
293
|
-
durationMs
|
|
274
|
+
durationMs,
|
|
294
275
|
}
|
|
295
276
|
}
|
|
296
277
|
|
|
@@ -311,18 +292,11 @@ export async function runMoAPipeline({
|
|
|
311
292
|
}
|
|
312
293
|
|
|
313
294
|
return {
|
|
314
|
-
kind: 'synthesis', content: single.text, aggregator: 'Fast Mode (Direct)',
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
usage: {
|
|
320
|
-
totalTokens, totalCostUsd,
|
|
321
|
-
candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
|
|
322
|
-
aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 },
|
|
323
|
-
},
|
|
324
|
-
skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [],
|
|
325
|
-
durationMs,
|
|
295
|
+
kind: 'synthesis', content: single.text, aggregator: 'Fast Mode (Direct)', references: referenceOutputs,
|
|
296
|
+
presetName: preset?.name || 'default', isRefinement, isFastMode: true, winningIndex: single.index,
|
|
297
|
+
winnerModel: single.label, promotedFiles, runId, ...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
298
|
+
usage: { totalTokens, totalCostUsd, candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })), aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 } },
|
|
299
|
+
skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs,
|
|
326
300
|
}
|
|
327
301
|
}
|
|
328
302
|
|
|
@@ -332,7 +306,25 @@ export async function runMoAPipeline({
|
|
|
332
306
|
}
|
|
333
307
|
|
|
334
308
|
// 7b. Consilium Round 2 (Peer Critique) if enabled
|
|
335
|
-
|
|
309
|
+
let runRound2 = isPeerCritiqueEnabled && successfulRefs.length > 1
|
|
310
|
+
if (runRound2 && preset?.budget_guard_enabled && preset?.max_budget_usd > 0) {
|
|
311
|
+
const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0)
|
|
312
|
+
const r2Next = estimateRound2Cost(successfulRefs, prices, maxTokens)
|
|
313
|
+
const downNext = estimateDownstreamCost(primaryJudge, preset?.judge_models, successfulRefs.length, prices, maxTokens)
|
|
314
|
+
const phaseCheck = checkPhaseBudget({ phaseName: 'Consilium Round 2', currentSpentUsd: currentSpent, nextPhaseCostUsd: r2Next + downNext, maxBudgetUsd: preset.max_budget_usd, action: preset.budget_action || 'trim', enabled: true })
|
|
315
|
+
if (!phaseCheck.allowed) {
|
|
316
|
+
if (phaseCheck.action === 'abort') {
|
|
317
|
+
await cleanMoaWorkspaces(cwd, { runId })
|
|
318
|
+
return createBudgetAbortPayload({ reason: phaseCheck.reason, primaryJudge, referenceOutputs, currentSpentUsd: currentSpent, preset, isRefinement, isFastMode, collectedCtx, startTime })
|
|
319
|
+
}
|
|
320
|
+
runRound2 = false
|
|
321
|
+
if (typeof onProgress === 'function') {
|
|
322
|
+
onProgress(`✂️ *Skipping Round 2 to stay within budget ($${preset.max_budget_usd.toFixed(4)})*\n`)
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
if (runRound2) {
|
|
336
328
|
if (typeof onProgress === 'function') {
|
|
337
329
|
onProgress('🤝 *Consilium Round 2: Candidates reviewing peer proposals in parallel...*\n')
|
|
338
330
|
}
|
|
@@ -371,8 +363,8 @@ export async function runMoAPipeline({
|
|
|
371
363
|
const extraCost = estimateTokenCost(slot, res.usage, prices)
|
|
372
364
|
cand.costUsd = Number(((cand.costUsd || 0) + extraCost.costUsd).toFixed(5))
|
|
373
365
|
}
|
|
374
|
-
} catch (
|
|
375
|
-
logger.warn(
|
|
366
|
+
} catch (r2Err) {
|
|
367
|
+
logger.warn(`[dsh-moa] Round 2 peer critique failed for candidate ${cand.index}:`, r2Err?.message || r2Err)
|
|
376
368
|
}
|
|
377
369
|
})
|
|
378
370
|
await Promise.allSettled(r2Promises)
|
|
@@ -412,6 +404,15 @@ export async function runMoAPipeline({
|
|
|
412
404
|
: null
|
|
413
405
|
|
|
414
406
|
// 8. Synthesis phase via primary judge or fallback chain
|
|
407
|
+
if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode) {
|
|
408
|
+
const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0) + (multiJudgeResult?.totalCostUsd || 0)
|
|
409
|
+
const downNext = estimateDownstreamCost(primaryJudge, [], successfulRefs.length, prices, maxTokens)
|
|
410
|
+
const phaseCheck = checkPhaseBudget({ phaseName: 'Judge Synthesis', currentSpentUsd: currentSpent, nextPhaseCostUsd: downNext, maxBudgetUsd: preset.max_budget_usd, action: preset.budget_action || 'trim', enabled: true })
|
|
411
|
+
if (!phaseCheck.allowed && fallbackJudges.length === 0) {
|
|
412
|
+
await cleanMoaWorkspaces(cwd, { runId })
|
|
413
|
+
return createBudgetAbortPayload({ reason: phaseCheck.reason, primaryJudge, referenceOutputs, currentSpentUsd: currentSpent, preset, isRefinement, isFastMode, collectedCtx, startTime })
|
|
414
|
+
}
|
|
415
|
+
}
|
|
415
416
|
const synthesisPrompt = buildSynthesisPrompt(userPrompt, referenceOutputs, judgeCriteria, {
|
|
416
417
|
curatorSynthesis: isCuratorSynthesis,
|
|
417
418
|
blindEvaluation: isBlindEvaluation,
|
|
@@ -429,6 +430,18 @@ export async function runMoAPipeline({
|
|
|
429
430
|
const currentJudge = judgesChain[jIdx]
|
|
430
431
|
const currentLabel = slotLabel(currentJudge)
|
|
431
432
|
|
|
433
|
+
if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode) {
|
|
434
|
+
const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0) + (multiJudgeResult?.totalCostUsd || 0)
|
|
435
|
+
const judgeCost = estimateDownstreamCost(currentJudge, [], successfulRefs.length, prices, maxTokens)
|
|
436
|
+
const phaseCheck = checkPhaseBudget({ phaseName: `Judge Synthesis (${currentLabel})`, currentSpentUsd: currentSpent, nextPhaseCostUsd: judgeCost, maxBudgetUsd: preset.max_budget_usd, action: preset.budget_action || 'trim', enabled: true })
|
|
437
|
+
if (!phaseCheck.allowed) {
|
|
438
|
+
lastJudgeError = new Error(phaseCheck.reason)
|
|
439
|
+
logger.warn(`[dsh-moa] Judge ${currentLabel} budget check failed: ${phaseCheck.reason}`)
|
|
440
|
+
if (typeof onProgress === 'function') onProgress(`⚠️ *Judge ${currentLabel} skipped: exceeds budget limit.*\n`)
|
|
441
|
+
continue
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
432
445
|
if (typeof onProgress === 'function') {
|
|
433
446
|
const modeTitle = isCuratorSynthesis ? 'Curator' : 'Judge'
|
|
434
447
|
const fallbackBadge = jIdx > 0 ? ` (Fallback #${jIdx})` : ''
|
|
@@ -473,6 +486,11 @@ export async function runMoAPipeline({
|
|
|
473
486
|
}
|
|
474
487
|
|
|
475
488
|
if (!judgeSuccess) {
|
|
489
|
+
if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode && (lastJudgeError?.message?.includes('exceeds max budget') || lastJudgeError?.message?.includes('Budget Guardrail') || preset?.budget_action === 'abort')) {
|
|
490
|
+
await cleanMoaWorkspaces(cwd, { runId })
|
|
491
|
+
const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0) + (multiJudgeResult?.totalCostUsd || 0)
|
|
492
|
+
return createBudgetAbortPayload({ reason: lastJudgeError?.message || 'Budget exceeded before judge synthesis', primaryJudge, referenceOutputs, currentSpentUsd: currentSpent, preset, isRefinement, isFastMode, collectedCtx, startTime })
|
|
493
|
+
}
|
|
476
494
|
const firstErr = lastJudgeError ? (lastJudgeError.message || String(lastJudgeError)) : 'unknown error'
|
|
477
495
|
synthesizedText = `⚠️ *[Aggregator error: ${firstErr}. Fallback candidate outputs:]*\n\n` +
|
|
478
496
|
successfulRefs.map((r, i) => `### Candidate ${i + 1} (${r.label})\n${r.text}`).join('\n\n')
|
|
@@ -548,46 +566,21 @@ export async function runMoAPipeline({
|
|
|
548
566
|
|
|
549
567
|
try {
|
|
550
568
|
await recordMoaRunAsync({
|
|
551
|
-
id: runId,
|
|
552
|
-
cwd,
|
|
553
|
-
prompt: userPrompt,
|
|
554
|
-
preset: preset?.name || 'default',
|
|
555
|
-
isRefinement,
|
|
569
|
+
id: runId, cwd, prompt: userPrompt, preset: preset?.name || 'default', isRefinement,
|
|
556
570
|
candidates: candidatesForHistory(referenceOutputs),
|
|
557
571
|
aggregator: { provider: chosenJudge.provider, model: chosenJudge.model, usage: aggUsage, costUsd: aggUsage.costUsd },
|
|
558
|
-
winnerIndex: winningIndex,
|
|
559
|
-
|
|
560
|
-
promotedFiles,
|
|
561
|
-
benchmarkReport,
|
|
562
|
-
consensus: multiJudgeResult,
|
|
563
|
-
totalTokens: usageSummary.totalTokens,
|
|
564
|
-
totalCostUsd: usageSummary.totalCostUsd,
|
|
565
|
-
durationMs,
|
|
572
|
+
winnerIndex: winningIndex, winnerModel, promotedFiles, benchmarkReport, consensus: multiJudgeResult,
|
|
573
|
+
totalTokens: usageSummary.totalTokens, totalCostUsd: usageSummary.totalCostUsd, durationMs,
|
|
566
574
|
}, historyFilePath || undefined)
|
|
567
575
|
} catch (histErr) {
|
|
568
576
|
logger.warn('[dsh-moa] Failed to record run in history:', histErr)
|
|
569
577
|
}
|
|
570
578
|
|
|
571
579
|
return {
|
|
572
|
-
kind: 'synthesis',
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
isRefinement,
|
|
578
|
-
isFastMode,
|
|
579
|
-
isCuratorSynthesis,
|
|
580
|
-
recommendedAssembler,
|
|
581
|
-
winningIndex,
|
|
582
|
-
winnerModel,
|
|
583
|
-
promotedFiles,
|
|
584
|
-
runId,
|
|
585
|
-
allowCandidateOverride,
|
|
586
|
-
benchmarkReport,
|
|
587
|
-
consensus: multiJudgeResult,
|
|
588
|
-
...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
589
|
-
usage: usageSummary,
|
|
590
|
-
skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [],
|
|
591
|
-
durationMs,
|
|
580
|
+
kind: 'synthesis', content: synthesizedText, aggregator: isFastMode ? 'Fast Mode (Direct)' : finalAggLabel,
|
|
581
|
+
references: referenceOutputs, presetName: preset?.name || 'default', isRefinement, isFastMode, isCuratorSynthesis,
|
|
582
|
+
recommendedAssembler, winningIndex, winnerModel, promotedFiles, runId, allowCandidateOverride,
|
|
583
|
+
benchmarkReport, consensus: multiJudgeResult, ...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
584
|
+
usage: usageSummary, skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs,
|
|
592
585
|
}
|
|
593
586
|
}
|
package/lib/moa-test-gate.js
CHANGED
|
@@ -5,6 +5,7 @@ import path from 'node:path'
|
|
|
5
5
|
import { spawn } from 'node:child_process'
|
|
6
6
|
import { resolveCandidateFolder } from './file-workspace.js'
|
|
7
7
|
import { bestEffort } from './best-effort.js'
|
|
8
|
+
import { logger } from './logger.js'
|
|
8
9
|
|
|
9
10
|
const MAX_CAPTURE_BYTES = 256 * 1024
|
|
10
11
|
|
|
@@ -42,6 +43,34 @@ export function resolveCandidateDir(baseDir, candidateIndex, options = {}) {
|
|
|
42
43
|
}
|
|
43
44
|
}
|
|
44
45
|
|
|
46
|
+
const UNIX_ROOT_PREFIX = /^\/(?:home|root|etc|var|tmp|opt|mnt|srv|media|proc|sys|dev|usr|bin|lib|lib64|run|sbin|boot|Users|Applications|Library|System|Volumes|private)(?:\/|$)/
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Validates that an executed script or command does not reference files outside allowed roots (#161, #225).
|
|
50
|
+
*/
|
|
51
|
+
function scanPathTokensForLeak(text, allowedRoots, executionDir) {
|
|
52
|
+
if (!text || typeof text !== 'string') return null
|
|
53
|
+
const sanitized = text
|
|
54
|
+
.replace(/https?:\/\/[^\s"'<>]+/gi, '')
|
|
55
|
+
.replace(/\b(?:node|file|npm|data):[^\s"'<>]+/gi, '')
|
|
56
|
+
const matches = sanitized.match(/(?:[a-zA-Z]:[\\/][^\s"'<>]+|(?<![\w.@\/\\])\/(?:[a-zA-Z0-9_\-.]+\/)*[a-zA-Z0-9_\-.]+|(?<![\w.@\/\\])\.\.[\\/][^\s"'<>]*)/g) || []
|
|
57
|
+
for (const raw of matches) {
|
|
58
|
+
if (/^\/(usr\/(bin|lib|include)|bin|lib|lib64|etc|dev)(\/|$)/.test(raw) && !/\.(txt|json|env|npmrc|key|ssh|md)/i.test(raw)) {
|
|
59
|
+
continue
|
|
60
|
+
}
|
|
61
|
+
const isAbs = path.isAbsolute(raw)
|
|
62
|
+
const isUp = raw.startsWith('..')
|
|
63
|
+
if (!isAbs && !isUp) continue
|
|
64
|
+
if (isAbs && process.platform !== 'win32' && !UNIX_ROOT_PREFIX.test(raw)) continue
|
|
65
|
+
const resolved = path.resolve(executionDir, raw)
|
|
66
|
+
const allowed = allowedRoots.some((r) => r && (resolved === r || resolved.startsWith(r + path.sep)))
|
|
67
|
+
if (!allowed) {
|
|
68
|
+
return raw
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return null
|
|
72
|
+
}
|
|
73
|
+
|
|
45
74
|
/**
|
|
46
75
|
* Executes a test command in a candidate sandbox or ephemeral test staging directory.
|
|
47
76
|
*/
|
|
@@ -63,6 +92,7 @@ export async function runCandidateTestGate({
|
|
|
63
92
|
|
|
64
93
|
const startTime = Date.now()
|
|
65
94
|
let stageDir = null
|
|
95
|
+
let fenceFile = null
|
|
66
96
|
let executionDir = candidateDir
|
|
67
97
|
|
|
68
98
|
try {
|
|
@@ -70,7 +100,6 @@ export async function runCandidateTestGate({
|
|
|
70
100
|
const candPkg = path.join(candidateDir, 'package.json')
|
|
71
101
|
if (fsSync.existsSync(basePkg) && !fsSync.existsSync(candPkg)) {
|
|
72
102
|
stageDir = await fs.mkdtemp(path.join(os.tmpdir(), `moa-test-stage-${candidateIndex}-`))
|
|
73
|
-
|
|
74
103
|
await fs.cp(cwd, stageDir, {
|
|
75
104
|
recursive: true,
|
|
76
105
|
filter: (src) => {
|
|
@@ -80,8 +109,6 @@ export async function runCandidateTestGate({
|
|
|
80
109
|
return parts[0] !== 'node_modules' && parts[0] !== '.git' && parts[0] !== '.moa'
|
|
81
110
|
},
|
|
82
111
|
})
|
|
83
|
-
|
|
84
|
-
// #162: Preserve ESM & CJS resolution by symlinking project node_modules
|
|
85
112
|
const baseNodeModules = path.join(cwd, 'node_modules')
|
|
86
113
|
const stageNodeModules = path.join(stageDir, 'node_modules')
|
|
87
114
|
if (fsSync.existsSync(baseNodeModules)) {
|
|
@@ -89,15 +116,24 @@ export async function runCandidateTestGate({
|
|
|
89
116
|
fs.symlink(baseNodeModules, stageNodeModules, process.platform === 'win32' ? 'junction' : 'dir')
|
|
90
117
|
)
|
|
91
118
|
}
|
|
92
|
-
|
|
93
119
|
await fs.cp(candidateDir, stageDir, { recursive: true })
|
|
94
120
|
executionDir = stageDir
|
|
95
121
|
}
|
|
96
122
|
|
|
97
|
-
const
|
|
98
|
-
.
|
|
99
|
-
.
|
|
100
|
-
.
|
|
123
|
+
const replacePlaceholderSafely = (cmd, placeholder, dirPath) => {
|
|
124
|
+
if (!cmd.includes(placeholder)) return cmd
|
|
125
|
+
let res = cmd.replace(new RegExp(`(["'])(.*?)${placeholder}(.*?)\\1`, 'g'), (m, q, pre, post) => `${q}${pre}${dirPath}${post}${q}`)
|
|
126
|
+
res = res.replace(new RegExp(`(?<=^|\\s)([^\\s"']*?)${placeholder}([^\\s"']*?)(?=\\s|$)`, 'g'), (m, pre, post) => {
|
|
127
|
+
const full = pre + dirPath + post
|
|
128
|
+
return full.includes(' ') ? `"${full}"` : full
|
|
129
|
+
})
|
|
130
|
+
return res
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
let resolvedCmd = trimmedCmd
|
|
134
|
+
resolvedCmd = replacePlaceholderSafely(resolvedCmd, '{candidateDir}', candidateDir)
|
|
135
|
+
resolvedCmd = replacePlaceholderSafely(resolvedCmd, '{baseDir}', cwd)
|
|
136
|
+
resolvedCmd = replacePlaceholderSafely(resolvedCmd, '{stageDir}', executionDir)
|
|
101
137
|
|
|
102
138
|
let exitCode = 0
|
|
103
139
|
let stdout = ''
|
|
@@ -114,36 +150,18 @@ export async function runCandidateTestGate({
|
|
|
114
150
|
stdout = customRes?.stdout || ''
|
|
115
151
|
stderr = customRes?.stderr || ''
|
|
116
152
|
} else {
|
|
117
|
-
const nodePath = path.join(cwd, 'node_modules')
|
|
118
|
-
const allowedReads = [
|
|
119
|
-
executionDir,
|
|
120
|
-
cwd,
|
|
121
|
-
...(candidateDir && candidateDir !== executionDir ? [candidateDir] : []),
|
|
122
|
-
...(process.platform === 'win32'
|
|
123
|
-
? [process.env.SYSTEMROOT || 'C:\\Windows', path.dirname(process.execPath)]
|
|
124
|
-
: ['/usr', '/lib', '/lib64', '/etc', path.resolve(path.dirname(process.execPath), '..')]),
|
|
125
|
-
].filter(Boolean)
|
|
126
|
-
const permOpts = `--permission ${allowedReads.map((p) => `--allow-fs-read=${p}`).join(' ')} --allow-fs-write=${executionDir}`
|
|
127
|
-
|
|
128
|
-
const env = getScrubbedEnv({
|
|
129
|
-
MOA_CANDIDATE_INDEX: String(candidateIndex),
|
|
130
|
-
MOA_CANDIDATE_DIR: candidateDir,
|
|
131
|
-
MOA_BASE_DIR: cwd,
|
|
132
|
-
...(runId ? { MOA_RUN_ID: String(runId) } : {}),
|
|
133
|
-
NODE_PATH: process.env.NODE_PATH ? `${nodePath}:${process.env.NODE_PATH}` : nodePath,
|
|
134
|
-
NODE_OPTIONS: process.env.NODE_OPTIONS ? `${process.env.NODE_OPTIONS} ${permOpts}` : permOpts,
|
|
135
|
-
})
|
|
136
|
-
|
|
137
153
|
const allowedRoots = [executionDir, cwd, ...(candidateDir && candidateDir !== executionDir ? [candidateDir] : [])]
|
|
138
154
|
const cmdTokens = resolvedCmd.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g) || []
|
|
155
|
+
|
|
156
|
+
// 1. Check command tokens for absolute path leaks
|
|
139
157
|
for (const rawToken of cmdTokens) {
|
|
140
158
|
const token = rawToken.replace(/^["']|["']$/g, '')
|
|
141
|
-
if (path.isAbsolute(token)) {
|
|
142
|
-
if (/^\/(usr\/(bin|lib|include)|bin|lib|lib64|etc)/.test(token) && !/\.(txt|json|env|npmrc|key|ssh|md)/i.test(token)) {
|
|
159
|
+
if (path.isAbsolute(token) || token.startsWith('..')) {
|
|
160
|
+
if (/^\/(usr\/(bin|lib|include)|bin|lib|lib64|etc|dev)(\/|$)/.test(token) && !/\.(txt|json|env|npmrc|key|ssh|md)/i.test(token)) {
|
|
143
161
|
continue
|
|
144
162
|
}
|
|
145
|
-
const
|
|
146
|
-
const allowed = allowedRoots.some((
|
|
163
|
+
const resolved = path.resolve(executionDir, token)
|
|
164
|
+
const allowed = allowedRoots.some((r) => r && (resolved === r || resolved.startsWith(r + path.sep)))
|
|
147
165
|
if (!allowed) {
|
|
148
166
|
const durationMs = Date.now() - startTime
|
|
149
167
|
return {
|
|
@@ -152,13 +170,169 @@ export async function runCandidateTestGate({
|
|
|
152
170
|
exitCode: 1,
|
|
153
171
|
durationMs,
|
|
154
172
|
command: resolvedCmd,
|
|
155
|
-
output:
|
|
173
|
+
output: `Access denied: test command specifies path outside workspace sandbox (${token})`,
|
|
156
174
|
summary: `FAIL (code 1, ${durationMs}ms)`,
|
|
157
175
|
}
|
|
158
176
|
}
|
|
159
177
|
}
|
|
160
178
|
}
|
|
161
179
|
|
|
180
|
+
// 2. Scan script files referenced by command or present in candidate workspace (#161)
|
|
181
|
+
const candFiles = fsSync.existsSync(executionDir) ? fsSync.readdirSync(executionDir, { recursive: true }) : []
|
|
182
|
+
const scriptTokens = new Set([
|
|
183
|
+
...cmdTokens.map((t) => t.replace(/^["']|["']$/g, '')),
|
|
184
|
+
...candFiles.map((f) => String(f)),
|
|
185
|
+
])
|
|
186
|
+
|
|
187
|
+
for (const scriptName of scriptTokens) {
|
|
188
|
+
const scriptPath = path.isAbsolute(scriptName) ? scriptName : path.join(executionDir, scriptName)
|
|
189
|
+
if (fsSync.existsSync(scriptPath) && !fsSync.statSync(scriptPath).isDirectory()) {
|
|
190
|
+
try {
|
|
191
|
+
const scriptText = fsSync.readFileSync(scriptPath, 'utf8')
|
|
192
|
+
const leaked = scanPathTokensForLeak(scriptText, allowedRoots, executionDir)
|
|
193
|
+
if (leaked) {
|
|
194
|
+
const durationMs = Date.now() - startTime
|
|
195
|
+
return {
|
|
196
|
+
candidateIndex,
|
|
197
|
+
passed: false,
|
|
198
|
+
exitCode: 1,
|
|
199
|
+
durationMs,
|
|
200
|
+
command: resolvedCmd,
|
|
201
|
+
output: `Access denied: test script specifies path outside workspace sandbox (${leaked})`,
|
|
202
|
+
summary: `FAIL (code 1, ${durationMs}ms)`,
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
} catch {}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// 3. Scan package.json scripts if package.json exists in executionDir (#161, #206)
|
|
210
|
+
const candPkgFile = path.join(executionDir, 'package.json')
|
|
211
|
+
if (fsSync.existsSync(candPkgFile)) {
|
|
212
|
+
try {
|
|
213
|
+
const pkgObj = JSON.parse(fsSync.readFileSync(candPkgFile, 'utf8'))
|
|
214
|
+
const scriptsContent = Object.values(pkgObj.scripts || {}).join('\n')
|
|
215
|
+
const leaked = scanPathTokensForLeak(scriptsContent, allowedRoots, executionDir)
|
|
216
|
+
if (leaked) {
|
|
217
|
+
const durationMs = Date.now() - startTime
|
|
218
|
+
return {
|
|
219
|
+
candidateIndex,
|
|
220
|
+
passed: false,
|
|
221
|
+
exitCode: 1,
|
|
222
|
+
durationMs,
|
|
223
|
+
command: resolvedCmd,
|
|
224
|
+
output: `Access denied: package.json test script specifies path outside workspace sandbox (${leaked})`,
|
|
225
|
+
summary: `FAIL (code 1, ${durationMs}ms)`,
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
} catch {}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// Setup isolated HOME & node roots
|
|
232
|
+
const rawHome = process.env.HOME
|
|
233
|
+
const isCustomHome = rawHome && (rawHome.startsWith(cwd) || rawHome.startsWith(executionDir) || rawHome.includes('fake-home') || rawHome.startsWith(os.tmpdir()))
|
|
234
|
+
let effectiveHome = isCustomHome ? rawHome : path.join(executionDir, '.moa-home')
|
|
235
|
+
if (!fsSync.existsSync(effectiveHome)) {
|
|
236
|
+
await bestEffort('create candidate test home', () => fs.mkdir(effectiveHome, { recursive: true }))
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const nodeRoot = path.resolve(path.dirname(process.execPath), '..')
|
|
240
|
+
const nodeBinDir = path.dirname(process.execPath)
|
|
241
|
+
const allowedReads = [
|
|
242
|
+
executionDir,
|
|
243
|
+
cwd,
|
|
244
|
+
effectiveHome,
|
|
245
|
+
...(candidateDir && candidateDir !== executionDir ? [candidateDir] : []),
|
|
246
|
+
nodeRoot,
|
|
247
|
+
nodeBinDir,
|
|
248
|
+
...(process.platform === 'win32'
|
|
249
|
+
? [process.env.SYSTEMROOT || 'C:\\Windows']
|
|
250
|
+
: ['/usr', '/lib', '/lib64', '/etc']),
|
|
251
|
+
].filter(Boolean)
|
|
252
|
+
|
|
253
|
+
// Create security fence module for child process containment (#161, #206)
|
|
254
|
+
fenceFile = path.join(executionDir, `.moa-fence-${Date.now()}.cjs`)
|
|
255
|
+
const fenceCode = `
|
|
256
|
+
const cp = require('node:child_process');
|
|
257
|
+
const path = require('node:path');
|
|
258
|
+
const allowed = ${JSON.stringify(allowedReads)};
|
|
259
|
+
const UNIX_ROOT_PREFIX = /^\\/(?:home|root|etc|var|tmp|opt|mnt|srv|media|proc|sys|dev|usr|bin|lib|lib64|run|sbin|boot|Users|Applications|Library|System|Volumes|private)(?:\\/|$)/;
|
|
260
|
+
function checkSinglePath(p) {
|
|
261
|
+
if (!p || typeof p !== 'string') return true;
|
|
262
|
+
if (/^\\/(usr\\/(bin|lib|include)|bin|lib|lib64|etc|dev)(\\/|$)/.test(p) && !/\\.(txt|json|env|npmrc|key|ssh|md)/i.test(p)) return true;
|
|
263
|
+
const isAbs = path.isAbsolute(p);
|
|
264
|
+
const isUp = p.startsWith('..');
|
|
265
|
+
if (!isAbs && !isUp) return true;
|
|
266
|
+
if (isAbs && process.platform !== 'win32' && !UNIX_ROOT_PREFIX.test(p)) return true;
|
|
267
|
+
const res = path.resolve(process.cwd(), p);
|
|
268
|
+
return allowed.some(root => root && (res === root || res.startsWith(root + path.sep)));
|
|
269
|
+
}
|
|
270
|
+
function scanStringForPaths(str) {
|
|
271
|
+
if (!str || typeof str !== 'string') return true;
|
|
272
|
+
if (!checkSinglePath(str)) return false;
|
|
273
|
+
const matches = str.match(/(?:[a-zA-Z]:[\\\\/][^\\s"'<>]+|(?<![\\w.@/\\\\])\\/(?:[a-zA-Z0-9_\\-.]+\\/)*[a-zA-Z0-9_\\-.]+|(?<![\\w.@/\\\\])\\.\\.[\\\\/][^\\s"'<>]*)/g) || [];
|
|
274
|
+
for (const m of matches) {
|
|
275
|
+
if (!checkSinglePath(m)) return false;
|
|
276
|
+
}
|
|
277
|
+
return true;
|
|
278
|
+
}
|
|
279
|
+
function guardArgs(cmd, args) {
|
|
280
|
+
if (cmd && !scanStringForPaths(String(cmd))) {
|
|
281
|
+
throw new Error('ERR_ACCESS_DENIED: child process command outside sandbox: ' + cmd);
|
|
282
|
+
}
|
|
283
|
+
if (Array.isArray(args)) {
|
|
284
|
+
for (const a of args) {
|
|
285
|
+
if (a && !scanStringForPaths(String(a))) {
|
|
286
|
+
throw new Error('ERR_ACCESS_DENIED: child process argument outside sandbox: ' + a);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
const origSpawn = cp.spawn;
|
|
292
|
+
cp.spawn = function(cmd, args, opts) {
|
|
293
|
+
guardArgs(cmd, args);
|
|
294
|
+
return origSpawn.apply(this, arguments);
|
|
295
|
+
};
|
|
296
|
+
const origSpawnSync = cp.spawnSync;
|
|
297
|
+
cp.spawnSync = function(cmd, args, opts) {
|
|
298
|
+
guardArgs(cmd, args);
|
|
299
|
+
return origSpawnSync.apply(this, arguments);
|
|
300
|
+
};
|
|
301
|
+
const origExecFile = cp.execFile;
|
|
302
|
+
cp.execFile = function(file, args, opts, cb) {
|
|
303
|
+
guardArgs(file, args);
|
|
304
|
+
return origExecFile.apply(this, arguments);
|
|
305
|
+
};
|
|
306
|
+
const origExecFileSync = cp.execFileSync;
|
|
307
|
+
cp.execFileSync = function(file, args, opts) {
|
|
308
|
+
guardArgs(file, args);
|
|
309
|
+
return origExecFileSync.apply(this, arguments);
|
|
310
|
+
};
|
|
311
|
+
const origExec = cp.exec;
|
|
312
|
+
cp.exec = function(cmd, opts, cb) {
|
|
313
|
+
guardArgs(cmd, [cmd]);
|
|
314
|
+
return origExec.apply(this, arguments);
|
|
315
|
+
};
|
|
316
|
+
const origExecSync = cp.execSync;
|
|
317
|
+
cp.execSync = function(cmd, opts) {
|
|
318
|
+
guardArgs(cmd, [cmd]);
|
|
319
|
+
return origExecSync.apply(this, arguments);
|
|
320
|
+
};
|
|
321
|
+
`
|
|
322
|
+
fsSync.writeFileSync(fenceFile, fenceCode, 'utf8')
|
|
323
|
+
|
|
324
|
+
const permOpts = `--permission ${allowedReads.map((p) => `--allow-fs-read=${p}`).join(' ')} --allow-fs-write=${executionDir} --allow-fs-write=${effectiveHome} --allow-child-process --require ${fenceFile}`
|
|
325
|
+
const nodePath = path.join(cwd, 'node_modules')
|
|
326
|
+
const env = getScrubbedEnv({
|
|
327
|
+
HOME: effectiveHome,
|
|
328
|
+
MOA_CANDIDATE_INDEX: String(candidateIndex),
|
|
329
|
+
MOA_CANDIDATE_DIR: candidateDir,
|
|
330
|
+
MOA_BASE_DIR: cwd,
|
|
331
|
+
...(runId ? { MOA_RUN_ID: String(runId) } : {}),
|
|
332
|
+
NODE_PATH: process.env.NODE_PATH ? `${nodePath}:${process.env.NODE_PATH}` : nodePath,
|
|
333
|
+
NODE_OPTIONS: process.env.NODE_OPTIONS ? `${process.env.NODE_OPTIONS} ${permOpts}` : permOpts,
|
|
334
|
+
})
|
|
335
|
+
|
|
162
336
|
const runPromise = new Promise((resolve) => {
|
|
163
337
|
let child
|
|
164
338
|
let timeoutHandle = null
|
|
@@ -268,6 +442,9 @@ export async function runCandidateTestGate({
|
|
|
268
442
|
summary: `ERROR (${err.message || 'unknown'})`,
|
|
269
443
|
}
|
|
270
444
|
} finally {
|
|
445
|
+
if (fenceFile) {
|
|
446
|
+
await bestEffort('moa-test-gate cleanup fenceFile', () => fs.rm(fenceFile, { force: true }))
|
|
447
|
+
}
|
|
271
448
|
if (stageDir) {
|
|
272
449
|
await bestEffort('moa-test-gate cleanup stageDir', () =>
|
|
273
450
|
fs.rm(stageDir, { recursive: true, force: true })
|