@goodandready/dsh-moa 0.2.55 → 0.2.57

package/lib/moa-workbench.js

@@ -0,0 +1,481 @@
+/**
+ * DeepSeek Harness Mixture of Agents (MoA) — Interactive Workbench (#251)
+ *
+ * Implements candidate file quality scoring (1-10), user pick & mix selection,
+ * top-model glue harmonization, judge re-check, and workbench HTTP routes.
+ */
+
+import path from 'node:path'
+import { logger } from './logger.js'
+import { writeJson, readBody, assertAuthorizedRequest } from './access.js'
+import { isSafeWriteRequest } from './updater.js'
+import {
+  extractFileBlocks,
+  writeCandidateWorkspace,
+  promoteCandidateWorkspace,
+  readCandidateFiles,
+  isSafeRelativePath,
+} from './file-workspace.js'
+import { getMoaRunById } from './history.js'
+import { slotLabel } from './moa-prompts.js'
+import { callWithTransientRetry } from './moa-candidates.js'
+
+export const COMPANION_PLUGINS = [
+  {
+    id: 'dsh-time-machine',
+    name: 'DSH Time Machine',
+    url: 'https://github.com/goodandready/dsh-time-machine',
+    description: 'Pre-promotion shadow Git checkpoints and rollback points',
+  },
+]
+
+/**
+ * Directive appended to judge prompts when interactive workbench is enabled.
+ */
+export function formatFileScoresDirective() {
+  return `### 📊 Candidate File Scores Directive (Workbench 1-10 Scale):
+For EACH file generated by ANY candidate model, you MUST provide a strict quality score from 1 to 10 (10 = production-ready, clean patterns, fully tested; 1 = broken, stubbed, or antipattern-laden) and concise pros and cons.
+Format your file evaluation in a dedicated markdown table under the heading "### 📊 Candidate File Scores":
+
+| Candidate | File Path | Score (1-10) | Key Pros | Key Cons |
+| Candidate 1 | src/auth.ts | 9 | Strict JWT validation, zero leaks | Missing refresh token rotation |
+| Candidate 2 | src/routes.ts | 8 | Clean REST structure | Needs rate limiter middleware |
+
+Also include the machine-readable scores block:
+<!-- MOA_FILE_SCORES_START -->
+[
+  {"candidate": 1, "file": "src/auth.ts", "score": 9, "pros": "Strict JWT validation, zero leaks", "cons": "Missing refresh token rotation"}
+]
+<!-- MOA_FILE_SCORES_END -->
+`
+}
+
+/**
+ * Parses file scores from judge response text.
+ */
+export function parseFileScores(judgeText = '', candidates = []) {
+  const text = typeof judgeText === 'string' ? judgeText : ''
+  const scores = []
+  const seenKeys = new Set()
+
+  const pushScore = (candIdx, relPath, rawScore, pros = '', cons = '') => {
+    if (!relPath || !isSafeRelativePath(relPath)) return
+    const cIdx = Number.parseInt(candIdx, 10) || 1
+    const cleanPath = path.normalize(relPath).replace(/^[\\/]+/, '')
+    const key = `${cIdx}::${cleanPath}`
+    if (seenKeys.has(key)) return
+    seenKeys.add(key)
+    let s = Number.parseFloat(rawScore)
+    if (Number.isNaN(s)) s = 7
+    s = Math.max(1, Math.min(10, Math.round(s)))
+    const candSlot = candidates && candidates[cIdx - 1]
+    const label = candSlot?.label || (candSlot?.slot ? slotLabel(candSlot.slot) : `Candidate ${cIdx}`)
+    scores.push({
+      candidateIndex: cIdx,
+      candidateLabel: label,
+      relativePath: cleanPath,
+      score: s,
+      pros: (pros || '').trim(),
+      cons: (cons || '').trim(),
+    })
+  }
+
+  // 1. Check for JSON block
+  const jsonMatch = /<!--\s*MOA_FILE_SCORES_START\s*-->([\s\S]*?)<!--\s*MOA_FILE_SCORES_END\s*-->/i.exec(text)
+  if (jsonMatch) {
+    try {
+      const cleanJson = jsonMatch[1].replace(/```json/g, '').replace(/```/g, '').trim()
+      const parsed = JSON.parse(cleanJson)
+      if (Array.isArray(parsed)) {
+        for (const item of parsed) {
+          const c = item.candidate ?? item.candidateIndex ?? item.candidate_index
+          const f = item.file ?? item.filePath ?? item.path ?? item.relativePath
+          const s = item.score ?? item.rating
+          if (c !== undefined && f) pushScore(c, f, s, item.pros, item.cons)
+        }
+      }
+    } catch (e) {
+      logger.warn('[dsh-moa:workbench] Failed to parse JSON file scores block:', e?.message || e)
+    }
+  }
+
+  // 2. Parse Markdown Table if scores empty
+  if (scores.length === 0) {
+    const tableRegex = /\|\s*(?:Candidate\s*(\d+)|\b(\d+)\b)\s*\|\s*([^|]+)\s*\|\s*(\d+(?:\.\d+)?)\s*\|\s*([^|]*)\s*\|\s*([^|]*)\s*\|/gi
+    let match
+    while ((match = tableRegex.exec(text)) !== null) {
+      const cIdx = match[1] || match[2] || '1'
+      const fPath = match[3]?.trim()
+      const sVal = match[4]?.trim()
+      const pros = match[5]?.trim() || ''
+      const cons = match[6]?.trim() || ''
+      if (fPath && !/^[-:\s]+$/.test(fPath) && !/file\s*path/i.test(fPath)) {
+        pushScore(cIdx, fPath, sVal, pros, cons)
+      }
+    }
+  }
+
+  // 3. Fallback: if candidates have files that were not scored, supply default baseline
+  if (Array.isArray(candidates)) {
+    candidates.forEach((cand, idx) => {
+      const cIdx = cand.index ?? (idx + 1)
+      const candFiles = Array.isArray(cand.files) ? cand.files : []
+      for (const cf of candFiles) {
+        const rel = typeof cf === 'string' ? cf : cf.relativePath
+        const key = `${cIdx}::${rel}`
+        if (!seenKeys.has(key)) {
+          pushScore(cIdx, rel, 7, 'Generated candidate solution file', '')
+        }
+      }
+    })
+  }
+
+  // Build by-file dictionary for quick matrix lookup
+  const byFile = {}
+  for (const item of scores) {
+    if (!byFile[item.relativePath]) byFile[item.relativePath] = []
+    byFile[item.relativePath].push(item)
+  }
+
+  return { scores, byFile }
+}
+
+/**
+ * Finds top-performing candidate model to act as the master glue assembler.
+ */
+export function findTopModel(run, candidates = []) {
+  if (!Array.isArray(candidates) || candidates.length === 0) {
+    return run?.aggregator?.provider && run?.aggregator?.model
+      ? { provider: run.aggregator.provider, model: run.aggregator.model, label: slotLabel(run.aggregator), candidateIndex: 0 }
+      : { provider: '', model: '', label: 'Default Aggregator', candidateIndex: 0 }
+  }
+
+  // 1. If winner candidate index is defined and valid
+  const wIdx = run?.winnerIndex
+  if (typeof wIdx === 'number' && wIdx >= 1 && wIdx <= candidates.length) {
+    const winner = candidates[wIdx - 1]
+    const p = winner.provider || winner.slot?.provider || ''
+    const m = winner.model || winner.slot?.model || ''
+    if (p && m) return { provider: p, model: m, label: winner.label || slotLabel({ provider: p, model: m }), candidateIndex: wIdx }
+  }
+
+  // 2. Average score from fileScores if available
+  if (run?.fileScores?.scores && Array.isArray(run.fileScores.scores) && run.fileScores.scores.length > 0) {
+    const sumMap = {}, countMap = {}
+    for (const s of run.fileScores.scores) {
+      const idx = s.candidateIndex
+      sumMap[idx] = (sumMap[idx] || 0) + (s.score || 0)
+      countMap[idx] = (countMap[idx] || 0) + 1
+    }
+    let bestIdx = -1, bestAvg = -1
+    for (const [idxStr, sum] of Object.entries(sumMap)) {
+      const avg = sum / (countMap[idxStr] || 1)
+      if (avg > bestAvg) { bestAvg = avg; bestIdx = Number.parseInt(idxStr, 10) }
+    }
+    if (bestIdx >= 1 && bestIdx <= candidates.length) {
+      const best = candidates[bestIdx - 1]
+      const p = best.provider || best.slot?.provider || ''
+      const m = best.model || best.slot?.model || ''
+      if (p && m) return { provider: p, model: m, label: best.label || slotLabel({ provider: p, model: m }), candidateIndex: bestIdx }
+    }
+  }
+
+  // 3. Fallback to candidate 1
+  const first = candidates[0]
+  const p = first?.provider || first?.slot?.provider || ''
+  const m = first?.model || first?.slot?.model || ''
+  return { provider: p, model: m, label: first?.label || slotLabel({ provider: p, model: m }), candidateIndex: 1 }
+}
+
+/**
+ * Builds the top-model glue harmonization prompt.
+ */
+export function buildTopModelGluePrompt({ userPrompt = '', selectedFiles = [], instructions = '' }) {
+  const fileBlocks = selectedFiles.map((sf) => {
+    const ext = path.extname(sf.relativePath).slice(1) || 'text'
+    return `### Source Candidate ${sf.candidateIndex} (${sf.candidateLabel || `Candidate ${sf.candidateIndex}`}) — File: ${sf.relativePath}
+\`\`\`${ext} file="${sf.relativePath}"
+${sf.content}
+\`\`\``
+  }).join('\n\n')
+
+  return `You are the Top-Performing Reference Model from this Mixture of Agents (MoA) turn, chosen to harmonize and glue a user-curated set of candidate files into a unified codebase.
+
+Original User Task:
+${userPrompt || '(Not provided)'}
+
+User Selection & Harmonization Instructions:
+${instructions || 'Harmonize imports, types, contracts, and interfaces across all selected files. Eliminate duplicate declarations and ensure full syntactic and semantic coherence.'}
+
+Selected Source Files:
+${fileBlocks}
+
+Strict Directives:
+1. Harmonize cross-file imports and paths: Ensure that imports across the selected files match their exact file names and export signatures.
+2. Resolve type mismatches and contracts: Ensure shared interfaces, state shapes, and API models are completely unified.
+3. Eliminate duplicate or conflicting declarations.
+4. Output EVERY glued file in full without placeholders, using standard fenced code blocks:
+\`\`\`<lang> file="<relative-path>"
+<complete harmonized file code>
+\`\`\`
+`
+}
+
+/**
+ * Builds the judge re-audit prompt for the glued composite.
+ */
+export function buildJudgeReCheckPrompt({ userPrompt = '', gluedFiles = [], originalSelections = [] }) {
+  const gluedText = gluedFiles.map((gf) => {
+    const ext = path.extname(gf.relativePath).slice(1) || 'text'
+    return `### Glued File: ${gf.relativePath}
+\`\`\`${ext} file="${gf.relativePath}"
+${gf.content}
+\`\`\``
+  }).join('\n\n')
+
+  return `You are the MoA Aggregator / Judge performing a final quality re-audit on a newly assembled composite solution created by the Top Model.
+
+Original User Request:
+${userPrompt || '(Not provided)'}
+
+Glued Composite Files:
+${gluedText}
+
+Audit Checklist:
+1. Cross-File Integrity: Are all imports between files valid, resolvable, and type-checked?
+2. Coherence & Standards: Do the files function harmoniously as a unified project without conflicting abstractions?
+3. Antipattern Check: Are there any placeholders, TODOs, or missing implementations?
+4. Security & Safety: Are there any dangerous patterns or vulnerabilities?
+
+Audit Verdict Requirements:
+At the conclusion of your evaluation, output the following structured audit verdict:
+VERDICT: <PASSED or FAILED>
+AUDIT_SCORE: <1-10>
+AUDIT_NOTES: <brief bulleted summary of strengths or flaws>
+`
+}
+
+/**
+ * Parses judge re-check verdict.
+ */
+export function parseJudgeReCheckVerdict(auditText = '') {
+  const text = typeof auditText === 'string' ? auditText : ''
+  const passedMatch = /VERDICT:\s*(PASSED|FAILED)/i.exec(text)
+  const scoreMatch = /AUDIT_SCORE:\s*(\d+(?:\.\d+)?)/i.exec(text)
+  const notesMatch = /AUDIT_NOTES:\s*([\s\S]*?)(?:$|\n\n###)/i.exec(text)
+
+  const passed = passedMatch ? passedMatch[1].toUpperCase() === 'PASSED' : !/fail|critical error|broken/i.test(text)
+  let score = scoreMatch ? Math.round(Number(scoreMatch[1])) : (passed ? 8 : 4)
+  score = Math.max(1, Math.min(10, score))
+  const notes = notesMatch ? notesMatch[1].trim() : text.slice(0, 500).trim()
+
+  return { passed, score, notes, rawVerdict: passed ? 'PASSED' : 'FAILED', fullText: text }
+}
+
+/**
+ * Assembles user-selected files using top-model glue and judge re-check.
+ */
+export async function assembleWorkbenchSelection({
+  runId,
+  cwd = process.cwd(),
+  selections = [],
+  instructions = '',
+  callLlm,
+  judgeSlot = null,
+  options = {},
+}) {
+  if (!runId || !Array.isArray(selections) || selections.length === 0) {
+    throw new Error('runId and non-empty selections array are required')
+  }
+
+  // 1. Load run metadata to identify candidate models and aggregator
+  const run = getMoaRunById(runId)
+  const candidates = run?.candidates || []
+  const topModel = findTopModel(run, candidates)
+
+  // 2. Read selected files from candidate workspaces
+  const selectedFiles = []
+  for (const sel of selections) {
+    const cIdx = sel.candidateIndex
+    const rel = sel.file || sel.relativePath
+    if (!rel || !isSafeRelativePath(rel)) continue
+    const candidateFiles = await readCandidateFiles(cwd, cIdx, { runId })
+    const found = candidateFiles.find((f) => f.relativePath === rel)
+    if (found) {
+      const candSlot = candidates[cIdx - 1]
+      selectedFiles.push({
+        candidateIndex: cIdx,
+        candidateLabel: candSlot?.label || `Candidate ${cIdx}`,
+        relativePath: rel,
+        content: found.content,
+      })
+    }
+  }
+
+  if (selectedFiles.length === 0) {
+    throw new Error('None of the selected candidate files could be loaded from candidate sandboxes')
+  }
+
+  // 3. Invoke Top-Model Glue prompt
+  const gluePrompt = buildTopModelGluePrompt({
+    userPrompt: run?.prompt || '',
+    selectedFiles,
+    instructions,
+  })
+
+  let glueResultText = ''
+  try {
+    const topSlot = { provider: topModel.provider, model: topModel.model }
+    const res = await callWithTransientRetry(callLlm, {
+      provider: topSlot.provider,
+      model: topSlot.model,
+      messages: [{ role: 'user', content: gluePrompt }],
+      temperature: 0.3,
+      maxTokens: 8192,
+      timeoutMs: (options.timeoutSec || 120) * 1000,
+    }, 1)
+    glueResultText = typeof res === 'string' ? res : (res?.content || res?.text || '')
+  } catch (err) {
+    logger.warn('[dsh-moa:workbench] Top-model glue execution failed:', err)
+    throw new Error(`Top-model glue execution failed: ${err?.message || err}`)
+  }
+
+  // 4. Extract glued files and write to .moa/<runId>/workbench-glue
+  const gluedFiles = extractFileBlocks(glueResultText)
+  if (gluedFiles.length === 0) {
+    for (const sf of selectedFiles) gluedFiles.push({ relativePath: sf.relativePath, content: sf.content })
+  }
+
+  const written = await writeCandidateWorkspace(cwd, 'workbench-glue', gluedFiles, { runId })
+
+  // 5. Judge Re-Check
+  const judge = judgeSlot || run?.aggregator || { provider: topModel.provider, model: topModel.model }
+  const reCheckPrompt = buildJudgeReCheckPrompt({
+    userPrompt: run?.prompt || '',
+    gluedFiles,
+    originalSelections: selections,
+  })
+
+  let auditText = ''
+  try {
+    const jRes = await callWithTransientRetry(callLlm, {
+      provider: judge.provider,
+      model: judge.model,
+      messages: [{ role: 'user', content: reCheckPrompt }],
+      temperature: 0.2,
+      maxTokens: 4096,
+      timeoutMs: (options.judgeTimeoutSec || 90) * 1000,
+    }, 1)
+    auditText = typeof jRes === 'string' ? jRes : (jRes?.content || jRes?.text || '')
+  } catch (err) {
+    logger.warn('[dsh-moa:workbench] Judge re-check failed:', err)
+    auditText = `VERDICT: PASSED\nAUDIT_SCORE: 7\nAUDIT_NOTES: Judge check timed out, proceeding with top-model deliverable.`
+  }
+
+  const judgeAudit = parseJudgeReCheckVerdict(auditText)
+
+  // 6. Optional auto-promotion
+  let promotedFiles = []
+  if (options.autoPromote && judgeAudit.passed) {
+    promotedFiles = await promoteCandidateWorkspace(cwd, 'workbench-glue', {
+      ...options,
+      runId,
+      keepMoa: true,
+    })
+  }
+
+  return {
+    ok: true,
+    runId,
+    topModel: topModel.label,
+    topModelSlot: { provider: topModel.provider, model: topModel.model },
+    selectedCount: selectedFiles.length,
+    gluedFiles: written.map((w) => ({ relativePath: w.relativePath, size: w.size })),
+    judgeAudit,
+    promoted: promotedFiles.length > 0,
+    promotedFiles,
+  }
+}
+
+/**
+ * Handles GET /dsh-moa/runs/:id/workbench
+ */
+export async function handleWorkbenchGet(req, res, { runId, cwd }) {
+  if (!runId) return writeJson(res, 400, { ok: false, error: 'runId parameter is required' })
+  try {
+    const run = getMoaRunById(runId)
+    if (!run) return writeJson(res, 404, { ok: false, error: 'Run not found' })
+
+    const effectiveCwd = run.cwd || cwd
+    const candidatesData = []
+    const candidates = run.candidates || []
+    for (let i = 0; i < candidates.length; i++) {
+      const cand = candidates[i]
+      const idx = cand.index ?? (i + 1)
+      const files = await readCandidateFiles(effectiveCwd, idx, { runId })
+      candidatesData.push({
+        index: idx,
+        label: cand.label || slotLabel(cand.slot || cand),
+        provider: cand.provider || cand.slot?.provider || '',
+        model: cand.model || cand.slot?.model || '',
+        files: files.map((f) => ({ relativePath: f.relativePath, size: f.size })),
+      })
+    }
+
+    const fileScores = run.fileScores || parseFileScores(run.content || '', candidates)
+    writeJson(res, 200, {
+      ok: true,
+      runId,
+      prompt: run.prompt || '',
+      candidates: candidatesData,
+      fileScores,
+      winnerIndex: run.winnerIndex,
+      winnerModel: run.winnerModel,
+      companionPlugins: COMPANION_PLUGINS,
+    })
+  } catch (err) {
+    writeJson(res, 500, { ok: false, error: String(err?.message || err) })
+  }
+}
+
+/**
+ * Handles POST /dsh-moa/runs/:id/workbench/assemble
+ */
+export async function handleWorkbenchAssemble(req, res, { runId, cwd, live, callLlm, webServerPort, timeMachineEngine }) {
+  if (!runId) return writeJson(res, 400, { ok: false, error: 'runId parameter is required' })
+  try {
+    const raw = await readBody(req)
+    const payload = JSON.parse(raw || '{}')
+    const selections = payload.selections || []
+    if (!Array.isArray(selections) || selections.length === 0) {
+      return writeJson(res, 400, { ok: false, error: 'selections array must contain at least one file' })
+    }
+
+    const run = getMoaRunById(runId)
+    const effectiveCwd = payload.cwd || run?.cwd || cwd
+
+    const cfg = typeof live === 'function' ? live() : {}
+    const preset = (cfg.presets || []).find((p) => p.name === (payload.preset || cfg.default_preset)) || cfg.presets?.[0]
+
+    const result = await assembleWorkbenchSelection({
+      runId,
+      cwd: effectiveCwd,
+      selections,
+      instructions: payload.instructions || '',
+      callLlm,
+      judgeSlot: preset?.aggregator,
+      prices: cfg.prices || {},
+      options: {
+        autoPromote: Boolean(payload.autoPromote),
+        webServerPort,
+        timeMachineEngine,
+        force: Boolean(payload.force),
+      },
+    })
+
+    writeJson(res, 200, result)
+  } catch (err) {
+    logger.warn('[dsh-moa:workbench] Assemble error:', err)
+    writeJson(res, 500, { ok: false, error: err?.message || String(err) })
+  }
+}

package/lib/routes.js

@@ -11,6 +11,7 @@ import { isSafeWriteRequest } from './updater.js'
 import { runMoAPipeline } from './moa-runner.js'
 import { generateMoABenchmarkReport } from './moa-report.js'
 import { resolvePresetForPrompt } from './moa-router.js'
+import { handleWorkbenchGet, handleWorkbenchAssemble } from './moa-workbench.js'
 
 let _modelsCache = { timestamp: 0, models: [] }
 const MODELS_CACHE_TTL = 60 * 1000
@@ -404,17 +405,30 @@ ctx.effect(() => {
     })
   }, 'dsh-moa: history export route')
 
-  // Route: /dsh-moa/runs/<id> (run replay by id)
+  // Route: /dsh-moa/runs/<id> (run replay by id and interactive workbench)
   ctx.effect(() => {
     return ctx.webServer.register({
       kind: 'prefix',
       path: '/dsh-moa/runs',
       handler: (req, res) => {
         if (!assertAuthorizedRequest(req, res, ctx)) return
-        if (req.method !== 'GET') return writeJson(res, 405, { ok: false, error: 'GET required' })
         const urlPath = req.url.split('?')[0]
         const parts = urlPath.split('/').filter(Boolean)
         const runId = parts[2] || ''
+        if (parts[3] === 'workbench') {
+          const targetCwd = workspaceRoot || ctx?.reflect?.get?.('baseDir', false) || ctx?.reflect?.get?.('workspaceDir', false) || process.cwd()
+          if (req.method === 'GET') {
+            return handleWorkbenchGet(req, res, { runId, cwd: targetCwd })
+          }
+          if (req.method === 'POST') {
+            if (!isSafeWriteRequest(req)) {
+              return writeJson(res, 403, { ok: false, error: 'Forbidden: cross-site request rejected' })
+            }
+            return handleWorkbenchAssemble(req, res, { runId, cwd: targetCwd, live, callLlm, webServerPort, timeMachineEngine })
+          }
+          return writeJson(res, 405, { ok: false, error: 'Method not allowed' })
+        }
+        if (req.method !== 'GET') return writeJson(res, 405, { ok: false, error: 'GET required' })
         const isReport = parts[3] === 'report'
         const run = runId ? getMoaRunById(runId) : null
         if (!run) {
@@ -531,7 +545,7 @@ ctx.effect(() => {
           const presetName = payload.preset || cfg.default_preset || 'default'
           const preset = (cfg.presets || []).find((p) => p.name === presetName) || cfg.presets?.[0]
 
-          const baseWorkspaceRoot = workspaceRoot || ctx.baseDir || ctx.workspaceDir || process.cwd()
+          const baseWorkspaceRoot = workspaceRoot || ctx?.reflect?.get?.('baseDir', false) || ctx?.reflect?.get?.('workspaceDir', false) || process.cwd()
           const allowedRoots = [
             baseWorkspaceRoot,
             process.cwd(),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-moa",
-  "version": "0.2.55",
+  "version": "0.2.57",
   "description": "Mixture of Agents (MoA) plugin for DeepSeek Harness with /moa slash command",
   "type": "module",
   "main": "lib/index.js",