@goodandready/dsh-moa 0.2.60 → 0.2.62

package/lib/pipeline/runner.js

@@ -0,0 +1,163 @@
+import crypto from 'node:crypto'
+import { isTestEnvironment } from '../history.js'
+import { cleanMoaWorkspaces } from '../workspace/fs.js'
+import { buildCompositeBlockDirectives } from '../moa-multi-judge.js'
+import { formatFileScoresDirective } from '../moa-workbench.js'
+import { stageProjectContext } from './context.js'
+import { stageCandidateRound1 } from './candidates.js'
+import { stageClarifyingQuestions } from './questions.js'
+import { stageFastMode } from './fast-mode.js'
+import { stagePeerCritique } from './peer-critique.js'
+import { stageMultiJudge } from './multi-judge.js'
+import { stageSynthesis } from './synthesis.js'
+import { stagePromotion } from './promotion.js'
+
+export async function runMoAPipeline({
+  userPrompt,
+  messages = [],
+  preset,
+  callLlm,
+  cwd = process.cwd(), prices = {}, onProgress = null,
+  historyFilePath = null, liveCanvas = null, onStreamDelta = null, signal = null,
+  testGateEnabled = null, testCommand = null, execFn = null,
+  webServerPort = null, timeMachineEngine = null, timeMachineUrl = null,
+  force = isTestEnvironment() ? true : false, checkpoint = true, checkpointFn = null, runId: customRunId = null,
+}) {
+  const runId = (customRunId && String(customRunId).trim()) || crypto.randomUUID()
+  const promoteOpts = { webServerPort, timeMachineEngine, timeMachineUrl, force: Boolean(force), checkpoint, checkpointFn, signal }
+  if (signal?.aborted) return { error: new Error('Turn aborted') }
+  const startTime = Date.now()
+
+  // Stage 1: Context & Configuration
+  const ctxStage = await stageProjectContext({
+    userPrompt, messages, preset, cwd, prices, maxTokens: preset?.max_tokens, startTime, onProgress,
+  })
+  if (!ctxStage.ok) return ctxStage.abortPayload
+
+  // Stage 2: Parallel Candidate Fan-out & Test Gate
+  const candStage = await stageCandidateRound1({
+    effectiveReferences: ctxStage.effectiveReferences,
+    enrichedMessages: ctxStage.enrichedMessages,
+    candidateSystemPrompt: ctxStage.candidateSystemPrompt,
+    refTemp: ctxStage.refTemp,
+    preset, maxTokens: ctxStage.maxTokens, prices,
+    refTimeoutSec: ctxStage.refTimeoutSec,
+    isQuorumEnabled: ctxStage.isQuorumEnabled,
+    gracePeriodSec: ctxStage.gracePeriodSec,
+    candidateRetries: ctxStage.candidateRetries,
+    signal, callLlm, onProgress, cwd, runId, startTime,
+    primaryJudge: ctxStage.primaryJudge,
+    isRefinement: ctxStage.isRefinement,
+    isFastMode: ctxStage.isFastMode,
+    collectedCtx: ctxStage.collectedCtx,
+    testGateEnabled, testCommand, execFn,
+  })
+  if (!candStage.ok) {
+    if (candStage.abort) return { error: candStage.error }
+    return candStage.failurePayload
+  }
+  const { referenceOutputs, successfulRefs } = candStage
+
+  // Stage 3: Clarifying Questions Branch
+  if (ctxStage.needsQuestions) {
+    return await stageClarifyingQuestions({
+      userPrompt, referenceOutputs, primaryJudge: ctxStage.primaryJudge,
+      aggTimeoutSec: ctxStage.aggTimeoutSec, prices, callLlm, cwd, runId,
+      preset, isRefinement: ctxStage.isRefinement, isFastMode: ctxStage.isFastMode,
+      collectedCtx: ctxStage.collectedCtx, startTime, historyFilePath, onProgress,
+    })
+  }
+
+  // Stage 4: Fast Mode (1 candidate bypass)
+  if (ctxStage.isFastMode) {
+    const fastRes = await stageFastMode({
+      successfulRefs, referenceOutputs,
+      allowCandidateOverride: ctxStage.allowCandidateOverride,
+      promoteOpts, cwd, runId, signal, userPrompt, preset,
+      isRefinement: ctxStage.isRefinement, historyFilePath, startTime,
+      collectedCtx: ctxStage.collectedCtx, liveCanvas,
+    })
+    if (!fastRes.ok) {
+      if (fastRes.abort) return { error: fastRes.error }
+      return fastRes.payload
+    }
+    return fastRes.payload
+  }
+
+  if (signal?.aborted) {
+    await cleanMoaWorkspaces(cwd, { runId })
+    return { error: new Error('Turn aborted') }
+  }
+
+  // Stage 5: Consilium Round 2 (Peer Critique)
+  const peerRes = await stagePeerCritique({
+    isPeerCritiqueEnabled: ctxStage.isPeerCritiqueEnabled,
+    successfulRefs, referenceOutputs, preset, prices,
+    maxTokens: ctxStage.maxTokens, primaryJudge: ctxStage.primaryJudge,
+    userPrompt, isBlindEvaluation: ctxStage.isBlindEvaluation,
+    refTemp: ctxStage.refTemp, refTimeoutSec: ctxStage.refTimeoutSec,
+    candidateRetries: ctxStage.candidateRetries, signal, callLlm, onProgress,
+    cwd, runId, isRefinement: ctxStage.isRefinement, isFastMode: ctxStage.isFastMode,
+    collectedCtx: ctxStage.collectedCtx, startTime,
+    options: { testGateEnabled, testCommand, execFn },
+  })
+  if (!peerRes.ok) return peerRes.abortPayload
+
+  if (signal?.aborted) {
+    await cleanMoaWorkspaces(cwd, { runId })
+    return { error: new Error('Turn aborted') }
+  }
+
+  // Stage 6: Multi-Judge Panel
+  const mjRes = await stageMultiJudge({
+    preset, successfulRefs, referenceOutputs, primaryJudge: ctxStage.primaryJudge,
+    userPrompt, judgeCriteria: ctxStage.judgeCriteria, prices, maxTokens: ctxStage.maxTokens,
+    callLlm, signal, onProgress, cwd, runId, isRefinement: ctxStage.isRefinement,
+    isFastMode: ctxStage.isFastMode, collectedCtx: ctxStage.collectedCtx, startTime,
+  })
+  if (!mjRes.ok) return mjRes.abortPayload
+  const multiJudgeResult = mjRes.multiJudgeResult
+
+  const compositeMergeDirective = (preset?.composite_merge_enabled && successfulRefs.length > 1)
+    ? buildCompositeBlockDirectives(successfulRefs, { blind: Boolean(preset.blind_evaluation) })
+    : null
+  const fileScoringDirective = (preset?.interactive_workbench_enabled)
+    ? formatFileScoresDirective()
+    : null
+
+  // Stage 7: Synthesis Phase
+  const synthRes = await stageSynthesis({
+    userPrompt, referenceOutputs, successfulRefs, judgeCriteria: ctxStage.judgeCriteria,
+    isCuratorSynthesis: ctxStage.isCuratorSynthesis, isBlindEvaluation: ctxStage.isBlindEvaluation,
+    priorTurnBaseline: ctxStage.priorTurnBaseline, compositeMergeDirective, fileScoringDirective,
+    multiJudgeResult, preset, prices, maxTokens: ctxStage.maxTokens, isFastMode: ctxStage.isFastMode,
+    primaryJudge: ctxStage.primaryJudge, fallbackJudges: ctxStage.fallbackJudges,
+    judgesChain: ctxStage.judgesChain, aggTemp: ctxStage.aggTemp,
+    aggTimeoutSec: ctxStage.aggTimeoutSec, isStreamAggregator: ctxStage.isStreamAggregator,
+    onStreamDelta, signal, callLlm, onProgress, cwd, runId, isRefinement: ctxStage.isRefinement,
+    collectedCtx: ctxStage.collectedCtx, startTime,
+  })
+  if (!synthRes.ok) return synthRes.abortPayload
+  const { synthesizedText, chosenJudge, aggUsage } = synthRes
+
+  if (signal?.aborted) {
+    await cleanMoaWorkspaces(cwd, { runId })
+    return { error: new Error('Turn aborted') }
+  }
+
+  // Stage 8: Evaluation, Winner & Promotion
+  const promRes = await stagePromotion({
+    synthesizedText, referenceOutputs, successfulRefs, multiJudgeResult,
+    isCuratorSynthesis: ctxStage.isCuratorSynthesis,
+    allowCandidateOverride: ctxStage.allowCandidateOverride,
+    promoteOpts, cwd, runId, signal, liveCanvas, startTime, chosenJudge, aggUsage,
+    userPrompt, preset, isRefinement: ctxStage.isRefinement, isFastMode: ctxStage.isFastMode,
+    effectiveReferences: ctxStage.effectiveReferences, referenceModels: ctxStage.referenceModels,
+    historyFilePath, collectedCtx: ctxStage.collectedCtx,
+  })
+  if (!promRes.ok) {
+    if (promRes.abort) return { error: promRes.error }
+    throw promRes.error
+  }
+  return promRes.payload
+}

package/lib/pipeline/synthesis.js

@@ -0,0 +1,150 @@
+import { logger } from '../logger.js'
+import { buildSynthesisPrompt, slotLabel } from '../moa-prompts.js'
+import { estimateDownstreamCost, checkPhaseBudget, createBudgetAbortPayload } from '../moa-budget.js'
+import { callWithTransientRetry } from '../moa-candidates.js'
+import { estimateTokenCost } from '../pricing.js'
+import { cleanMoaWorkspaces } from '../workspace/fs.js'
+
+export async function stageSynthesis({
+  userPrompt,
+  referenceOutputs,
+  successfulRefs,
+  judgeCriteria,
+  isCuratorSynthesis,
+  isBlindEvaluation,
+  priorTurnBaseline,
+  compositeMergeDirective,
+  fileScoringDirective,
+  multiJudgeResult,
+  preset,
+  prices,
+  maxTokens,
+  isFastMode,
+  primaryJudge,
+  fallbackJudges,
+  judgesChain,
+  aggTemp,
+  aggTimeoutSec,
+  isStreamAggregator,
+  onStreamDelta,
+  signal,
+  callLlm,
+  onProgress,
+  cwd,
+  runId,
+  isRefinement,
+  collectedCtx,
+  startTime,
+}) {
+  const synthesisPrompt = buildSynthesisPrompt(userPrompt, referenceOutputs, judgeCriteria, {
+    curatorSynthesis: isCuratorSynthesis,
+    blindEvaluation: isBlindEvaluation,
+    priorTurnBaseline,
+    compositeMergeDirective,
+    fileScoringDirective,
+    consensusReport: multiJudgeResult?.consensusReport,
+  })
+  const synthPromptTokens = Math.max(50, Math.ceil(synthesisPrompt.length / 4))
+
+  if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode) {
+    const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0) + (multiJudgeResult?.totalCostUsd || 0)
+    const downNext = estimateDownstreamCost(primaryJudge, [], successfulRefs.length, prices, maxTokens, synthPromptTokens)
+    const phaseCheck = checkPhaseBudget({
+      phaseName: 'Judge Synthesis', currentSpentUsd: currentSpent, nextPhaseCostUsd: downNext,
+      maxBudgetUsd: preset.max_budget_usd, action: preset.budget_action || 'trim', enabled: true,
+    })
+    if (!phaseCheck.allowed && fallbackJudges.length === 0) {
+      await cleanMoaWorkspaces(cwd, { runId })
+      const abortPayload = createBudgetAbortPayload({
+        reason: phaseCheck.reason, primaryJudge, referenceOutputs, currentSpentUsd: currentSpent,
+        preset, isRefinement, isFastMode, collectedCtx, startTime, multiJudgeResult,
+      })
+      return { ok: false, abortPayload }
+    }
+  }
+
+  let synthesizedText = ''
+  let aggUsage = { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 }
+  let chosenJudge = primaryJudge
+  let judgeSuccess = false
+  let lastJudgeError = null
+
+  for (let jIdx = 0; jIdx < judgesChain.length; jIdx++) {
+    const currentJudge = judgesChain[jIdx]
+    const currentLabel = slotLabel(currentJudge)
+
+    if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode) {
+      const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0) + (multiJudgeResult?.totalCostUsd || 0)
+      const judgeCost = estimateDownstreamCost(currentJudge, [], successfulRefs.length, prices, maxTokens, synthPromptTokens)
+      const phaseCheck = checkPhaseBudget({
+        phaseName: `Judge Synthesis (${currentLabel})`, currentSpentUsd: currentSpent, nextPhaseCostUsd: judgeCost,
+        maxBudgetUsd: preset.max_budget_usd, action: preset.budget_action || 'trim', enabled: true,
+      })
+      if (!phaseCheck.allowed) {
+        lastJudgeError = new Error(phaseCheck.reason)
+        logger.warn(`[dsh-moa] Judge ${currentLabel} budget check failed: ${phaseCheck.reason}`)
+        if (typeof onProgress === 'function') onProgress(`⚠️ *Judge ${currentLabel} skipped: exceeds budget limit.*\n`)
+        continue
+      }
+    }
+
+    if (typeof onProgress === 'function') {
+      const modeTitle = isCuratorSynthesis ? 'Curator' : 'Judge'
+      const fallbackBadge = jIdx > 0 ? ` (Fallback #${jIdx})` : ''
+      onProgress(`⚖️ *${modeTitle} (${currentLabel}${fallbackBadge}) synthesizing solution...*\n`)
+    }
+
+    try {
+      const aggRes = await callWithTransientRetry(callLlm, {
+        provider: currentJudge.provider,
+        model: currentJudge.model,
+        messages: [{ role: 'user', content: synthesisPrompt }],
+        temperature: aggTemp,
+        maxTokens,
+        timeoutMs: aggTimeoutSec * 1000,
+        signal,
+        onStreamDelta: (delta) => {
+          if (isStreamAggregator && typeof onStreamDelta === 'function') {
+            onStreamDelta(delta)
+          }
+        },
+      }, 0)
+
+      synthesizedText = typeof aggRes === 'string' ? aggRes : (aggRes?.content || aggRes?.text || '')
+
+      const aggFallbackUsage = (typeof aggRes === 'object' && aggRes?.usage) ? aggRes.usage : {
+        inputTokens: Math.max(1, Math.round(synthesisPrompt.length / 4)),
+        outputTokens: Math.max(1, Math.round(synthesizedText.length / 4)),
+      }
+      aggUsage = estimateTokenCost(currentJudge, aggFallbackUsage, prices)
+      chosenJudge = currentJudge
+      judgeSuccess = true
+      break
+    } catch (err) {
+      lastJudgeError = err
+      logger.warn(`[dsh-moa] Judge ${currentLabel} failed:`, err)
+      if (typeof onProgress === 'function') {
+        const nextJudge = judgesChain[jIdx + 1]
+        const nextHint = nextJudge ? ` Trying fallback ${slotLabel(nextJudge)}...` : ''
+        onProgress(`⚠️ *Judge ${currentLabel} failed: ${err?.message || err}.${nextHint}*\n`)
+      }
+    }
+  }
+
+  if (!judgeSuccess) {
+    if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode && (lastJudgeError?.message?.includes('exceeds max budget') || lastJudgeError?.message?.includes('Budget Guardrail') || preset?.budget_action === 'abort')) {
+      await cleanMoaWorkspaces(cwd, { runId })
+      const currentSpent = referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0) + (multiJudgeResult?.totalCostUsd || 0)
+      const abortPayload = createBudgetAbortPayload({
+        reason: lastJudgeError?.message || 'Budget exceeded before judge synthesis', primaryJudge, referenceOutputs,
+        currentSpentUsd: currentSpent, preset, isRefinement, isFastMode, collectedCtx, startTime, multiJudgeResult,
+      })
+      return { ok: false, abortPayload }
+    }
+    const firstErr = lastJudgeError ? (lastJudgeError.message || String(lastJudgeError)) : 'unknown error'
+    synthesizedText = `⚠️ *[Aggregator error: ${firstErr}. Fallback candidate outputs:]*\n\n` +
+      successfulRefs.map((r, i) => `### Candidate ${i + 1} (${r.label})\n${r.text}`).join('\n\n')
+  }
+
+  return { ok: true, synthesizedText, chosenJudge, aggUsage }
+}

package/lib/routes/history.js

@@ -0,0 +1,81 @@
+import { getMoaHistory, getMoaLeaderboard, exportMoaHistory } from '../history.js'
+import { writeJson, assertAuthorizedRequest } from '../access.js'
+
+export function registerHistoryRoutes(ctx, { live }) {
+  ctx.effect(() => {
+    return ctx.webServer.register({
+      kind: 'exact',
+      path: '/dsh-moa/history',
+      handler: async (req, res) => {
+        if (!assertAuthorizedRequest(req, res, ctx)) return
+        if (req.method !== 'GET') return writeJson(res, 405, { ok: false, error: 'GET required' })
+        try {
+          const u = new URL(req.url, 'http://localhost')
+          const limit = Math.max(1, Math.min(100, parseInt(u.searchParams.get('limit') || '20', 10) || 20))
+          const offset = Math.max(0, parseInt(u.searchParams.get('offset') || '0', 10) || 0)
+          const data = getMoaHistory(limit, offset)
+          writeJson(res, 200, { ok: true, ...data })
+        } catch (err) { writeJson(res, 500, { ok: false, error: err?.message || String(err) }) }
+      },
+    })
+  }, 'dsh-moa: history route')
+
+  // Route: /dsh-moa/leaderboard
+
+  ctx.effect(() => {
+    return ctx.webServer.register({
+      kind: 'exact',
+      path: '/dsh-moa/leaderboard',
+      handler: async (req, res) => {
+        if (!assertAuthorizedRequest(req, res, ctx)) return
+        if (req.method !== 'GET') return writeJson(res, 405, { ok: false, error: 'GET required' })
+        try {
+          const url = new URL(req.url || '', 'http://127.0.0.1')
+          const preset = url.searchParams.get('preset') || null
+          const format = url.searchParams.get('format') || 'json'
+
+          if (format === 'csv') {
+            const csv = exportMoaHistory(undefined, { format: 'csv', preset })
+            res.writeHead(200, {
+              'Content-Type': 'text/csv; charset=utf-8',
+              'Content-Disposition': 'attachment; filename="moa-leaderboard.csv"',
+            })
+            res.end(csv)
+            return
+          }
+
+          const data = getMoaLeaderboard(undefined, { preset })
+          writeJson(res, 200, { ok: true, ...data })
+        } catch (err) { writeJson(res, 500, { ok: false, error: err?.message || String(err) }) }
+      },
+    })
+  }, 'dsh-moa: leaderboard route')
+
+  // Route: /dsh-moa/promote (Candidate Override Action)
+
+  ctx.effect(() => {
+    return ctx.webServer.register({
+      kind: 'exact',
+      path: '/dsh-moa/history/export',
+      handler: async (req, res) => {
+        if (!assertAuthorizedRequest(req, res, ctx)) return
+        if (req.method !== 'GET') return writeJson(res, 405, { ok: false, error: 'GET required' })
+        try {
+          const url = new URL(req.url || '', 'http://127.0.0.1')
+          const preset = url.searchParams.get('preset') || null
+          const format = url.searchParams.get('format') || 'json'
+          const data = exportMoaHistory(undefined, { format, preset })
+
+          const isCsv = format === 'csv'
+          res.writeHead(200, {
+            'Content-Type': isCsv ? 'text/csv; charset=utf-8' : 'application/json; charset=utf-8',
+            'Content-Disposition': `attachment; filename="moa-history.${isCsv ? 'csv' : 'json'}"`,
+          })
+          res.end(data)
+        } catch (err) { writeJson(res, 500, { ok: false, error: err?.message || String(err) }) }
+      },
+    })
+  }, 'dsh-moa: history export route')
+
+  // Route: /dsh-moa/runs/<id> (run replay by id and interactive workbench)
+}

package/lib/routes/pipeline.js

@@ -0,0 +1,91 @@
+import path from 'node:path'
+import os from 'node:os'
+import { isTestEnvironment } from '../history.js'
+import { isLoopbackUrl } from '../file-workspace.js'
+import { runMoAPipeline } from '../moa-runner.js'
+import { writeJson, readBody, isAllowedWorkspace, assertAuthorizedRequest } from '../access.js'
+import { isSafeWriteRequest } from '../updater.js'
+
+
+export function registerPipelineRoutes(ctx, { live, callLlm, liveCanvas, webServerPort, timeMachineEngine, workspaceRoot }) {
+  let _lastRunTimestamp = 0
+  const RUN_MIN_INTERVAL_MS = 1000
+
+  ctx.effect(() => {
+    return ctx.webServer.register({
+      kind: 'exact',
+      path: '/dsh-moa/run',
+      handler: async (req, res) => {
+        if (!assertAuthorizedRequest(req, res, ctx)) return
+        if (req.method !== 'POST') return writeJson(res, 405, { ok: false, error: 'POST required' })
+        if (!isSafeWriteRequest(req)) {
+          writeJson(res, 403, { ok: false, error: 'Forbidden: cross-site request rejected' })
+          return
+        }
+        const now = Date.now()
+        if (now - _lastRunTimestamp < RUN_MIN_INTERVAL_MS) {
+          writeJson(res, 429, { ok: false, error: 'Rate limit exceeded: please wait before starting another run' })
+          return
+        }
+
+        try {
+          const raw = await readBody(req)
+          const payload = JSON.parse(raw)
+          const prompt = payload.prompt || ''
+          if (!prompt.trim()) {
+            writeJson(res, 400, { ok: false, error: 'Empty prompt' })
+            return
+          }
+          _lastRunTimestamp = now
+
+          const cfg = live()
+          if (cfg.enabled === false) {
+            writeJson(res, 400, { ok: false, error: 'MoA is disabled in settings' })
+            return
+          }
+          if (payload.timeMachineUrl && !isLoopbackUrl(payload.timeMachineUrl)) {
+            writeJson(res, 400, { ok: false, error: 'Invalid timeMachineUrl: loopback host required' })
+            return
+          }
+          const presetName = payload.preset || cfg.default_preset || 'default'
+          const preset = (cfg.presets || []).find((p) => p.name === presetName) || cfg.presets?.[0]
+
+          const baseWorkspaceRoot = workspaceRoot || ctx?.reflect?.get?.('baseDir', false) || ctx?.reflect?.get?.('workspaceDir', false) || process.cwd()
+          const allowedRoots = [
+            baseWorkspaceRoot,
+            process.cwd(),
+            cfg.workspace_root,
+            ...(Array.isArray(cfg.allowed_workspaces) ? cfg.allowed_workspaces : (cfg.allowed_workspaces ? [cfg.allowed_workspaces] : [])),
+            ...(isTestEnvironment() ? [os.tmpdir()] : []),
+          ].filter(Boolean)
+
+          const targetCwd = payload.cwd ? path.resolve(payload.cwd) : baseWorkspaceRoot
+          if (payload.cwd && !isAllowedWorkspace(targetCwd, allowedRoots)) {
+            writeJson(res, 403, { ok: false, error: 'Forbidden: cwd escapes allowed workspace boundary' })
+            return
+          }
+
+          const result = await runMoAPipeline({
+            userPrompt: prompt, messages: payload.messages || [], preset, callLlm,
+            cwd: targetCwd, prices: cfg.prices || {}, liveCanvas,
+            webServerPort: webServerPort || ctx.webServer?.port,
+            timeMachineEngine: timeMachineEngine || ctx.reflect?.get?.('timeMachineEngine', false),
+            timeMachineUrl: payload.timeMachineUrl, force: false,
+            checkpoint: true,
+            runId: payload.runId,
+          })
+
+          if (result?.skippedFiles > 0) {
+            ctx.logger?.warn?.(`[dsh-moa] Workspace scan skipped ${result.skippedFiles} file(s): ${result.skippedList.join(', ')}`)
+          }
+          if (result?.error) {
+            writeJson(res, 500, { ok: false, error: result.error?.message || String(result.error) })
+            return
+          }
+
+          writeJson(res, 200, { ok: true, ...result })
+        } catch (err) { writeJson(res, 500, { ok: false, error: err?.message || String(err) }) }
+      },
+    })
+  }, 'dsh-moa: run pipeline route')
+}