@goodandready/dsh-moa 0.2.70 → 0.2.72

package/lib/pipeline/early-consensus.js

@@ -0,0 +1,324 @@
+/**
+ * Early Exit upon Semantic Consensus (#299)
+ * Detects convergence between reference candidates and bypasses judge round when consensus >= threshold.
+ * Saves up to 60-80% tokens and latency on requests where candidates converge.
+ */
+
+import { logger } from '../logger.js'
+import { promoteCandidateWorkspace, cleanMoaWorkspaces } from '../workspace/fs.js'
+import { createPromotedPreview } from '../live-canvas.js'
+import { recordMoaRunAsync, candidatesForHistory } from '../history.js'
+import { emitEvent, MOA_EVENTS } from '../events.js'
+import { estimateTokenCost } from '../pricing.js'
+
+/**
+ * Normalizes text/code by stripping comments and normalizing whitespace for consensus comparison.
+ * @param {string} text
+ * @returns {string}
+ */
+export function normalizeForConsensus(text = '') {
+  if (typeof text !== 'string') return ''
+  return text
+    // Strip block comments /* ... */
+    .replace(/\/\*[\s\S]*?\*\//g, '')
+    // Strip JS/TS single line comments // ...
+    .replace(/\/\/.*$/gm, '')
+    // Strip Python/Shell comments # ...
+    .replace(/(^|\s)#.*$/gm, '$1')
+    // Normalize newlines
+    .replace(/\r?\n/g, '\n')
+    // Collapse horizontal whitespace
+    .replace(/[ \t]+/g, ' ')
+    // Collapse multi-blank lines
+    .replace(/\n\s*\n/g, '\n')
+    .trim()
+}
+
+/**
+ * Calculates Sørensen-Dice similarity coefficient between two code/text snippets (0.0 to 1.0).
+ * @param {string} textA
+ * @param {string} textB
+ * @returns {number}
+ */
+export function calculateSimilarity(textA = '', textB = '') {
+  const normA = normalizeForConsensus(textA)
+  const normB = normalizeForConsensus(textB)
+
+  if (normA === normB) return 1.0
+  if (!normA || !normB) return 0.0
+
+  const tokensA = normA.match(/[\w$]+|[^\s\w]/g) || []
+  const tokensB = normB.match(/[\w$]+|[^\s\w]/g) || []
+
+  if (tokensA.length === 0 && tokensB.length === 0) return 1.0
+  if (tokensA.length === 0 || tokensB.length === 0) return 0.0
+
+  const freqA = new Map()
+  for (const t of tokensA) freqA.set(t, (freqA.get(t) || 0) + 1)
+
+  const freqB = new Map()
+  for (const t of tokensB) freqB.set(t, (freqB.get(t) || 0) + 1)
+
+  let intersection = 0
+  for (const [token, countA] of freqA.entries()) {
+    const countB = freqB.get(token) || 0
+    intersection += Math.min(countA, countB)
+  }
+
+  const dice = (2 * intersection) / (tokensA.length + tokensB.length)
+  return Number(dice.toFixed(4))
+}
+
+/**
+ * Checks if candidates have reached semantic consensus above threshold.
+ * @param {Array<object>} candidates
+ * @param {number} [threshold=0.95]
+ * @returns {{ reached: boolean, similarity: number, winnerIndex: number, consensusIndices: number[], winnerCandidate: object | null, reason: string }}
+ */
+export function checkSemanticConsensus(candidates = [], threshold = 0.95) {
+  if (!Array.isArray(candidates) || candidates.length < 2) {
+    return {
+      reached: false,
+      similarity: 0,
+      winnerIndex: 0,
+      consensusIndices: [],
+      winnerCandidate: null,
+      reason: 'Insufficient candidates for consensus check',
+    }
+  }
+
+  const n = candidates.length
+  let maxSim = 0
+  let bestPair = [0, 1]
+  const matrix = Array.from({ length: n }, () => new Float32Array(n))
+
+  for (let i = 0; i < n; i++) {
+    matrix[i][i] = 1.0
+    for (let j = i + 1; j < n; j++) {
+      const sim = calculateSimilarity(candidates[i].text || '', candidates[j].text || '')
+      matrix[i][j] = sim
+      matrix[j][i] = sim
+      if (sim > maxSim) {
+        maxSim = sim
+        bestPair = [i, j]
+      }
+    }
+  }
+
+  if (maxSim < threshold) {
+    return {
+      reached: false,
+      similarity: maxSim,
+      winnerIndex: 0,
+      consensusIndices: [],
+      winnerCandidate: null,
+      reason: `Max pairwise similarity ${maxSim} is below consensus threshold ${threshold}`,
+    }
+  }
+
+  // Find consensus cluster matching threshold with the best pair
+  const clusterSet = new Set(bestPair)
+  for (let k = 0; k < n; k++) {
+    if (k === bestPair[0] || k === bestPair[1]) continue
+    if (matrix[k][bestPair[0]] >= threshold && matrix[k][bestPair[1]] >= threshold) {
+      clusterSet.add(k)
+    }
+  }
+
+  const consensusIndices = Array.from(clusterSet).map((idx) => candidates[idx].index ?? idx)
+  const clusterCandidates = Array.from(clusterSet).map((idx) => candidates[idx])
+
+  // Select winner from consensus cluster (prioritize valid syntax, then lowest error count)
+  let winner = clusterCandidates[0]
+  for (const c of clusterCandidates) {
+    if (c.syntaxValidation?.valid && !winner.syntaxValidation?.valid) {
+      winner = c
+    }
+  }
+
+  return {
+    reached: true,
+    similarity: maxSim,
+    winnerIndex: winner.index ?? 1,
+    consensusIndices,
+    winnerCandidate: winner,
+    reason: `Semantic consensus reached (${maxSim} >= ${threshold}) across candidates ${consensusIndices.join(', ')}`,
+  }
+}
+
+/**
+ * Estimates saved tokens and cost from bypassing the judge round.
+ * @param {Array<object>} candidates
+ * @param {string} prompt
+ * @param {number} [maxTokens=4096]
+ * @param {object} [prices={}]
+ * @returns {{ skippedTokens: number, skippedCostUsd: number }}
+ */
+export function estimateSkippedJudgeUsage(candidates = [], prompt = '', maxTokens = 4096, prices = {}) {
+  const promptChars = (prompt?.length || 0) + candidates.reduce((sum, c) => sum + (c.text?.length || 0), 0)
+  const estimatedInputTokens = Math.max(100, Math.round(promptChars / 3.8))
+  const estimatedOutputTokens = Math.min(Math.max(200, Math.round(maxTokens * 0.25)), 1000)
+  const skippedTokens = estimatedInputTokens + estimatedOutputTokens
+
+  const costEst = estimateTokenCost({ provider: 'default', model: 'judge' }, {
+    inputTokens: estimatedInputTokens,
+    outputTokens: estimatedOutputTokens,
+  }, prices)
+
+  return {
+    skippedTokens,
+    skippedCostUsd: Number((costEst?.costUsd || 0.005).toFixed(5)),
+  }
+}
+
+/**
+ * Evaluates candidates for early consensus and executes early exit bypass if reached.
+ * @param {object} params
+ * @returns {Promise<{ ok: boolean, consensusReached: boolean, payload?: object, error?: Error, abort?: boolean }>}
+ */
+export function stageEarlyConsensus({
+  successfulRefs = [],
+  referenceOutputs = [],
+  preset = {},
+  userPrompt = '',
+  prices = {},
+  maxTokens = 4096,
+  cwd = process.cwd(),
+  runId = '',
+  signal = null,
+  promoteOpts = {},
+  allowCandidateOverride = false,
+  isRefinement = false,
+  historyFilePath = null,
+  startTime = Date.now(),
+  collectedCtx = null,
+  liveCanvas = null,
+  ctx = null,
+}) {
+  const enabled = Boolean(preset?.early_exit_consensus_enabled)
+  const threshold = typeof preset?.early_exit_threshold === 'number' ? preset.early_exit_threshold : 0.95
+
+  if (!enabled || successfulRefs.length < 2) {
+    return Promise.resolve({ ok: true, consensusReached: false })
+  }
+
+  const consensus = checkSemanticConsensus(successfulRefs, threshold)
+  if (!consensus.reached || !consensus.winnerCandidate) {
+    return Promise.resolve({ ok: true, consensusReached: false, similarity: consensus.similarity })
+  }
+
+  const winner = consensus.winnerCandidate
+  emitEvent(ctx, MOA_EVENTS.EARLY_CONSENSUS, {
+    runId,
+    similarity: consensus.similarity,
+    threshold,
+    winnerIndex: consensus.winnerIndex,
+    winnerModel: winner.label,
+    consensusCandidates: consensus.consensusIndices,
+  })
+
+  // Promote winning candidate files
+  let promotedFiles = []
+  let promoteErr = null
+  const runPromotion = async () => {
+    if (winner.files?.length > 0) {
+      try {
+        promotedFiles = await promoteCandidateWorkspace(cwd, winner.index, {
+          ...promoteOpts,
+          runId,
+          keepMoa: allowCandidateOverride,
+        })
+      } catch (err) {
+        promoteErr = err
+      }
+    } else if (!allowCandidateOverride) {
+      await cleanMoaWorkspaces(cwd, { runId })
+    }
+
+    if (promoteErr) {
+      if (!allowCandidateOverride) await cleanMoaWorkspaces(cwd, { runId })
+      if (signal?.aborted || promoteErr?.message?.includes('aborted')) {
+        return { ok: false, consensusReached: true, abort: true, error: new Error('Turn aborted') }
+      }
+    }
+
+    const skippedUsage = estimateSkippedJudgeUsage(successfulRefs, userPrompt, maxTokens, prices)
+    const livePreview = await createPromotedPreview(liveCanvas, cwd, promotedFiles, referenceOutputs, {
+      winningIndex: winner.index,
+    })
+
+    const totalTokens = referenceOutputs.reduce((acc, r) => acc + (r.usage?.totalTokens || 0), 0)
+    const totalCostUsd = Number(referenceOutputs.reduce((acc, r) => acc + (r.costUsd || 0), 0).toFixed(5))
+    const durationMs = Date.now() - startTime
+
+    try {
+      await recordMoaRunAsync({
+        id: runId,
+        cwd,
+        prompt: userPrompt,
+        preset: preset?.name || 'default',
+        isRefinement,
+        isEarlyConsensus: true,
+        candidates: candidatesForHistory(referenceOutputs),
+        aggregator: 'Early Consensus (Bypassed)',
+        winnerIndex: winner.index,
+        winnerModel: winner.label,
+        promotedFiles,
+        totalTokens,
+        totalCostUsd,
+        skippedJudgeTokens: skippedUsage.skippedTokens,
+        durationMs,
+      }, historyFilePath || undefined)
+    } catch (histErr) {
+      logger.warn('[dsh-moa] Failed to record early consensus in history:', histErr?.message || histErr)
+    }
+
+    emitEvent(ctx, MOA_EVENTS.RUN_FINISH, {
+      runId,
+      durationMs,
+      totalTokens,
+      totalCostUsd,
+      promotedFiles,
+      winnerIndex: winner.index,
+      isEarlyConsensus: true,
+    })
+
+    return {
+      ok: true,
+      consensusReached: true,
+      payload: {
+        kind: 'synthesis',
+        content: winner.text,
+        aggregator: 'Early Consensus (Bypassed)',
+        references: referenceOutputs,
+        presetName: preset?.name || 'default',
+        isRefinement,
+        isEarlyConsensus: true,
+        winningIndex: winner.index,
+        winnerModel: winner.label,
+        promotedFiles,
+        runId,
+        consensusInfo: {
+          similarity: consensus.similarity,
+          threshold,
+          consensusIndices: consensus.consensusIndices,
+        },
+        ...(livePreview ? { liveCanvas: livePreview } : {}),
+        usage: {
+          totalTokens,
+          totalCostUsd,
+          skippedJudgeTokens: skippedUsage.skippedTokens,
+          skippedJudgeCostUsd: skippedUsage.skippedCostUsd,
+          candidates: referenceOutputs.map((r) => ({ label: r.label, usage: r.usage, costUsd: r.costUsd })),
+          aggregator: { inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 },
+        },
+        skippedFiles: collectedCtx?.skippedFiles || 0,
+        skippedList: collectedCtx?.skippedList || [],
+        durationMs,
+      },
+    }
+  }
+
+  return runPromotion()
+}
+

package/lib/pipeline/runner.js

@@ -1,3 +1,4 @@
+import { buildAstCompositeMergeDirective } from '../moa-ast-merge.js'
 import { runVisualMoaConsensus } from '../moa-visual-consensus.js'
 import crypto from 'node:crypto'
 import { isTestEnvironment } from '../history.js'
@@ -12,6 +13,8 @@ import { stagePeerCritique } from './peer-critique.js'
 import { stageMultiJudge } from './multi-judge.js'
 import { stageSynthesis } from './synthesis.js'
 import { stagePromotion } from './promotion.js'
+import { applySyntaxFilter, formatSyntaxDirective } from './syntax-filter.js'
+import { stageEarlyConsensus } from './early-consensus.js'
 import { emitEvent, MOA_EVENTS } from '../events.js'
 
 export async function runMoAPipeline({
@@ -76,7 +79,7 @@ export async function runMoAPipeline({
     })
     return candStage.failurePayload
   }
-  const { referenceOutputs, successfulRefs } = candStage
+  const { referenceOutputs, successfulRefs, adversarialTddResult } = candStage
 
   for (const ref of referenceOutputs) {
     emitEvent(ctx, MOA_EVENTS.CANDIDATE_COMPLETE, {
@@ -89,6 +92,16 @@ export async function runMoAPipeline({
     })
   }
 
+  // Pre-Judge Stage 2.5: Zero-Token Local Syntax Gate (#303)
+  const syntaxResult = applySyntaxFilter(successfulRefs, { enabled: ctxStage.localPreflightCheck })
+  emitEvent(ctx, MOA_EVENTS.SYNTAX_VALIDATION, {
+    runId,
+    filterApplied: syntaxResult.filterApplied,
+    validCount: syntaxResult.validCandidates.length,
+    invalidCount: syntaxResult.invalidCandidates.length,
+  })
+  const syntaxDirective = syntaxResult.filterApplied ? formatSyntaxDirective(referenceOutputs, { blind: ctxStage.isBlindEvaluation }) : null
+
   // Stage 3: Clarifying Questions Branch
   if (ctxStage.needsQuestions) {
     return await stageClarifyingQuestions({
@@ -102,7 +115,8 @@ export async function runMoAPipeline({
   // Stage 4: Fast Mode (1 candidate bypass)
   if (ctxStage.isFastMode) {
     const fastRes = await stageFastMode({
-      successfulRefs, referenceOutputs,
+      successfulRefs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
+      referenceOutputs,
       allowCandidateOverride: ctxStage.allowCandidateOverride,
       promoteOpts, cwd, runId, signal, userPrompt, preset,
       isRefinement: ctxStage.isRefinement, historyFilePath, startTime,
@@ -115,6 +129,32 @@ export async function runMoAPipeline({
     return fastRes.payload
   }
 
+  // Pre-Judge Stage 4.5: Early Exit upon Semantic Consensus (#299)
+  if (ctxStage.earlyExitConsensusEnabled && successfulRefs.length > 1) {
+    const earlyRes = await stageEarlyConsensus({
+      successfulRefs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
+      referenceOutputs,
+      preset,
+      userPrompt,
+      prices,
+      maxTokens: ctxStage.maxTokens,
+      cwd,
+      runId,
+      signal,
+      promoteOpts,
+      allowCandidateOverride: ctxStage.allowCandidateOverride,
+      isRefinement: ctxStage.isRefinement,
+      historyFilePath,
+      startTime,
+      collectedCtx: ctxStage.collectedCtx,
+      liveCanvas,
+      ctx,
+    })
+    if (earlyRes.consensusReached) {
+      return earlyRes.payload
+    }
+  }
+
   if (signal?.aborted) {
     await cleanMoaWorkspaces(cwd, { runId })
     return { error: new Error('Turn aborted') }
@@ -123,7 +163,8 @@ export async function runMoAPipeline({
   // Stage 5: Consilium Round 2 (Peer Critique)
   const peerRes = await stagePeerCritique({
     isPeerCritiqueEnabled: ctxStage.isPeerCritiqueEnabled,
-    successfulRefs, referenceOutputs, preset, prices,
+    successfulRefs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
+    referenceOutputs, preset, prices,
     maxTokens: ctxStage.maxTokens, primaryJudge: ctxStage.primaryJudge,
     userPrompt, isBlindEvaluation: ctxStage.isBlindEvaluation,
     refTemp: ctxStage.refTemp, refTimeoutSec: ctxStage.refTimeoutSec,
@@ -147,7 +188,9 @@ export async function runMoAPipeline({
 
   // Stage 6: Multi-Judge Panel
   const mjRes = await stageMultiJudge({
-    preset, successfulRefs, referenceOutputs, primaryJudge: ctxStage.primaryJudge,
+    preset,
+    successfulRefs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
+    referenceOutputs, primaryJudge: ctxStage.primaryJudge,
     userPrompt, judgeCriteria: ctxStage.judgeCriteria, prices, maxTokens: ctxStage.maxTokens,
     callLlm, signal, onProgress, cwd, runId, isRefinement: ctxStage.isRefinement,
     isFastMode: ctxStage.isFastMode, collectedCtx: ctxStage.collectedCtx, startTime,
@@ -155,9 +198,13 @@ export async function runMoAPipeline({
   if (!mjRes.ok) return mjRes.abortPayload
   const multiJudgeResult = mjRes.multiJudgeResult
 
-  const compositeMergeDirective = (preset?.composite_merge_enabled && successfulRefs.length > 1)
-    ? buildCompositeBlockDirectives(successfulRefs, { blind: Boolean(preset.blind_evaluation) })
-    : null
+  let compositeMergeDirective = null
+  if (preset?.ast_composite_merge_enabled && successfulRefs.length > 1) {
+    const astMerge = await buildAstCompositeMergeDirective(successfulRefs, { ctx, blind: Boolean(preset.blind_evaluation) })
+    compositeMergeDirective = astMerge.directive
+  } else if (preset?.composite_merge_enabled && successfulRefs.length > 1) {
+    compositeMergeDirective = buildCompositeBlockDirectives(successfulRefs, { blind: Boolean(preset.blind_evaluation) })
+  }
   let visualResult = null
   if (preset?.visual_moa_enabled) {
     visualResult = await runVisualMoaConsensus({
@@ -171,11 +218,13 @@ export async function runMoAPipeline({
   const rawFileDirective = (preset?.interactive_workbench_enabled)
     ? formatFileScoresDirective()
     : null
-  const fileScoringDirective = [rawFileDirective, visualResult?.directive].filter(Boolean).join("\n\n") || null
+  const fileScoringDirective = [rawFileDirective, visualResult?.directive, adversarialTddResult?.directive, syntaxDirective].filter(Boolean).join("\n\n") || null
 
   // Stage 7: Synthesis Phase
   const synthRes = await stageSynthesis({
-    userPrompt, referenceOutputs, successfulRefs, judgeCriteria: ctxStage.judgeCriteria,
+    userPrompt, referenceOutputs,
+    successfulRefs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
+    judgeCriteria: ctxStage.judgeCriteria,
     isCuratorSynthesis: ctxStage.isCuratorSynthesis, isBlindEvaluation: ctxStage.isBlindEvaluation,
     priorTurnBaseline: ctxStage.priorTurnBaseline, compositeMergeDirective, fileScoringDirective,
     multiJudgeResult, preset, prices, maxTokens: ctxStage.maxTokens, isFastMode: ctxStage.isFastMode,
@@ -195,7 +244,9 @@ export async function runMoAPipeline({
 
   // Stage 8: Evaluation, Winner & Promotion
   const promRes = await stagePromotion({
-    synthesizedText, referenceOutputs, successfulRefs, multiJudgeResult,
+    synthesizedText, referenceOutputs,
+    successfulRefs: syntaxResult.validCandidates.length > 0 ? syntaxResult.validCandidates : successfulRefs,
+    multiJudgeResult,
     isCuratorSynthesis: ctxStage.isCuratorSynthesis,
     allowCandidateOverride: ctxStage.allowCandidateOverride,
     promoteOpts, cwd, runId, signal, liveCanvas, startTime, chosenJudge, aggUsage,
@@ -209,3 +260,4 @@ export async function runMoAPipeline({
   }
   return promRes.payload
 }
+