@goodandready/dsh-moa 0.2.66 → 0.2.67

package/lib/index.js

@@ -1,3 +1,4 @@
+import { applyTuiSettings, buildTuiSection } from './tui-settings.js'
 import { registerMoaCommands } from './commands.js'
 import { service, getWebServerPort } from './services.js'
 import crypto from 'node:crypto'
@@ -189,6 +190,9 @@ export function apply(ctx, config) {
   // Official slash command /moa registration via dsh-commands (#275)
   registerMoaCommands(ctx, { live, moaService })
 
+  // Official TUI settings section registration (#277)
+  applyTuiSettings(ctx, { live })
+
   // Session turn interceptor: route turn to real aggregator model & stream MoA synthesis
   ctx.effect(() => {
     return () => {
@@ -391,3 +395,9 @@ export {
   getActiveLlmProviders,
   getActiveLlmModels,
 } from './moa-model-validator.js'
+export { applyTuiSettings, buildTuiSection } from './tui-settings.js'
+export {
+  calculateTokenEntropy,
+  analyzeCandidateUncertainty,
+  formatUncertaintyAnnotations,
+} from './moa-uncertainty.js'

package/lib/moa-candidates.js

@@ -147,6 +147,7 @@ export async function runReferencesParallel(references, messages, options = {},
           costUsd: costInfo.costUsd,
           durationMs: Date.now() - candidateStart,
           model: slot?.model ? (slot.provider ? (slot.provider + ":" + slot.model) : slot.model) : label,
+          ...(res?.logprobs ? { logprobs: res.logprobs } : {}),
           ok: true,
         }
 

package/lib/moa-schema.js

@@ -63,12 +63,17 @@ export const PresetSchema = z.object({
   local_fallback_enabled: z.boolean().default(false),
   local_fallback_models: z.array(ModelSlotSchema).default([]),
   interactive_workbench_enabled: z.boolean().default(false),
+  uncertainty_scoring_enabled: z.boolean().default(false),
 })
 
 export const Config = z.object({
   enabled: z.boolean().default(true).volatile(),
   default_preset: z.string().default('default').volatile(),
   smart_routing_enabled: z.boolean().default(false).volatile(),
+  interactive_workbench_enabled: z.boolean().default(false).volatile(),
+  ask_clarifying_questions: z.boolean().default(true).volatile(),
+  test_gate_enabled: z.boolean().default(false).volatile(),
+  uncertainty_scoring_enabled: z.boolean().default(false).volatile(),
   smart_routing_model: ModelSlotSchema.default({ provider: '', model: '' }).volatile(),
   prices: z.dict(PriceRow).default({}).volatile(),
   presets: z.array(PresetSchema).default([

package/lib/moa-uncertainty.js

@@ -0,0 +1,154 @@
+/**
+ * Token-Level Uncertainty & Entropy Scoring for Hallucination Detection (#256)
+ * Calculates Shannon entropy across token logprob distributions, detects high-uncertainty
+ * identifier/method call spans, and builds diagnostic annotations for the Judge.
+ */
+
+const LN2 = Math.LN2
+
+/**
+ * Computes token entropy in bits from a token logprob entry.
+ * Supports full top_logprobs distributions as well as scalar logprob surprisal.
+ *
+ * @param {object|number} entry
+ * @returns {number} Entropy in bits (>= 0)
+ */
+export function calculateTokenEntropy(entry) {
+  if (typeof entry === 'number') {
+    if (!Number.isFinite(entry) || entry >= 0) return 0
+    return Number((-entry / LN2).toFixed(4))
+  }
+  if (!entry || typeof entry !== 'object') return 0
+
+  const topList = Array.isArray(entry.top_logprobs) ? entry.top_logprobs : null
+  if (topList && topList.length > 0) {
+    let entropy = 0
+    let totalProb = 0
+    for (const item of topList) {
+      const lp = typeof item === 'number' ? item : Number(item?.logprob)
+      if (!Number.isFinite(lp)) continue
+      const p = Math.min(1, Math.max(1e-9, Math.exp(lp)))
+      totalProb += p
+      entropy -= p * (lp / LN2)
+    }
+    // Account for remaining probability mass if top_logprobs is truncated
+    if (totalProb < 0.999 && totalProb > 0) {
+      const rem = 1 - totalProb
+      entropy -= rem * Math.log2(rem)
+    }
+    return Number(Math.max(0, entropy).toFixed(4))
+  }
+
+  const lp = Number(entry.logprob)
+  if (Number.isFinite(lp) && lp < 0) {
+    return Number((-lp / LN2).toFixed(4))
+  }
+  return 0
+}
+
+/**
+ * Analyzes a candidate output's token logprobs (and code heuristics) to identify
+ * high-uncertainty zones where hallucinations are likely.
+ *
+ * @param {object} candidate
+ * @param {object} [options]
+ * @returns {object}
+ */
+export function analyzeCandidateUncertainty(candidate, options = {}) {
+  const threshold = typeof options.entropyThreshold === 'number' ? options.entropyThreshold : 1.2
+  const rawLogprobs = Array.isArray(candidate?.logprobs)
+    ? candidate.logprobs
+    : (Array.isArray(candidate?.logprobs?.content) ? candidate.logprobs.content : [])
+
+  const highEntropySpans = []
+  let sumEntropy = 0
+  let maxEntropy = 0
+  let tokenCount = 0
+
+  if (rawLogprobs.length > 0) {
+    for (let i = 0; i < rawLogprobs.length; i++) {
+      const item = rawLogprobs[i]
+      const h = calculateTokenEntropy(item)
+      const tok = String(item?.token ?? '').trim()
+      sumEntropy += h
+      if (h > maxEntropy) maxEntropy = h
+      tokenCount++
+
+      if (h >= threshold && tok.length > 0 && !/^[\s{}()\[\];,.:'"`]+$/.test(tok)) {
+        // Reconstruct local context window around token
+        const prev = rawLogprobs.slice(Math.max(0, i - 2), i).map((t) => t?.token || '').join('')
+        const next = rawLogprobs.slice(i + 1, Math.min(rawLogprobs.length, i + 3)).map((t) => t?.token || '').join('')
+        const contextSpan = (prev + (item?.token || '') + next).replace(/\s+/g, ' ').trim()
+        const prob = Number.isFinite(Number(item?.logprob))
+          ? Number(Math.exp(Number(item.logprob)).toFixed(3))
+          : Number(Math.pow(2, -h).toFixed(3))
+
+        highEntropySpans.push({
+          token: tok,
+          span: contextSpan || tok,
+          entropy: h,
+          probability: prob,
+          warning: `Внимание: высокая энтропия на участке "${contextSpan || tok}" (H=${h.toFixed(2)} bits, p=${prob})`,
+        })
+      }
+    }
+  }
+
+  const avgEntropy = tokenCount > 0 ? Number((sumEntropy / tokenCount).toFixed(4)) : 0
+  let riskLevel = 'low'
+  if (maxEntropy >= threshold * 1.6 || highEntropySpans.length >= 3) {
+    riskLevel = 'high'
+  } else if (maxEntropy >= threshold || highEntropySpans.length >= 1) {
+    riskLevel = 'medium'
+  }
+
+  return {
+    candidateIndex: candidate?.index ?? 0,
+    model: candidate?.label || candidate?.model || 'unknown',
+    tokenCount,
+    avgEntropy,
+    maxEntropy: Number(maxEntropy.toFixed(4)),
+    riskLevel,
+    highEntropySpans,
+  }
+}
+
+/**
+ * Formats uncertainty analysis across all candidates into a Judge prompt directive.
+ *
+ * @param {Array<object>} candidates
+ * @param {object} [options]
+ * @returns {{ reportText: string, analyses: Array<object>, hasHighUncertainty: boolean }}
+ */
+export function formatUncertaintyAnnotations(candidates, options = {}) {
+  if (!Array.isArray(candidates) || candidates.length === 0) {
+    return { reportText: '', analyses: [], hasHighUncertainty: false }
+  }
+
+  const analyses = candidates.map((c, idx) =>
+    analyzeCandidateUncertainty({ ...c, index: c.index ?? idx + 1 }, options)
+  )
+
+  const flagged = analyses.filter((a) => a.highEntropySpans.length > 0)
+  if (flagged.length === 0) {
+    return { reportText: '', analyses, hasHighUncertainty: false }
+  }
+
+  const lines = [
+    '\n### 🔬 Token-Level Uncertainty & Hallucination Warnings:',
+    'The following candidate segments exhibited abnormally high token entropy (low model confidence). Carefully verify API existence, method signatures, and edge cases before adopting these fragments:',
+  ]
+
+  for (const a of flagged) {
+    lines.push(`- **Candidate ${a.candidateIndex} (${a.model})** [Risk: ${a.riskLevel.toUpperCase()}, Avg H=${a.avgEntropy}, Max H=${a.maxEntropy}]:`)
+    for (const span of a.highEntropySpans.slice(0, 5)) {
+      lines.push(`  - ⚠️ ${span.warning}`)
+    }
+  }
+
+  return {
+    reportText: lines.join('\n') + '\n',
+    analyses,
+    hasHighUncertainty: true,
+  }
+}

package/lib/pipeline/synthesis.js

@@ -1,3 +1,4 @@
+import { formatUncertaintyAnnotations } from '../moa-uncertainty.js'
 import { logger } from '../logger.js'
 import { buildSynthesisPrompt, slotLabel } from '../moa-prompts.js'
 import { estimateDownstreamCost, checkPhaseBudget, createBudgetAbortPayload } from '../moa-budget.js'
@@ -36,7 +37,14 @@ export async function stageSynthesis({
   collectedCtx,
   startTime,
 }) {
-  const synthesisPrompt = buildSynthesisPrompt(userPrompt, referenceOutputs, judgeCriteria, {
+  let uncertaintyDirective = ''
+  if (preset?.uncertainty_scoring_enabled) {
+    const uncReport = formatUncertaintyAnnotations(referenceOutputs)
+    if (uncReport.hasHighUncertainty) {
+      uncertaintyDirective = uncReport.reportText
+    }
+  }
+  const baseSynthesisPrompt = buildSynthesisPrompt(userPrompt, referenceOutputs, judgeCriteria, {
     curatorSynthesis: isCuratorSynthesis,
     blindEvaluation: isBlindEvaluation,
     priorTurnBaseline,
@@ -44,6 +52,7 @@ export async function stageSynthesis({
     fileScoringDirective,
     consensusReport: multiJudgeResult?.consensusReport,
   })
+  const synthesisPrompt = uncertaintyDirective ? baseSynthesisPrompt + '\n' + uncertaintyDirective : baseSynthesisPrompt
   const synthPromptTokens = Math.max(50, Math.ceil(synthesisPrompt.length / 4))
 
   if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode) {

package/lib/tui-settings.js

@@ -0,0 +1,146 @@
+/**
+ * DSH Terminal User Interface (TUI) Settings Section for dsh-moa (#277)
+ */
+
+import { service } from './services.js'
+
+function boolFormat(value) {
+  return value === false ? 'false' : 'true'
+}
+
+function boolParse(text) {
+  return { kind: 'set', value: String(text).trim() === 'true' }
+}
+
+/**
+ * Builds the TUI settings section descriptor from live configuration.
+ *
+ * @param {{ live: () => Record<string, any> }} deps
+ * @returns {object}
+ */
+export function buildTuiSection({ live }) {
+  const cfg = (typeof live === 'function' ? live() : live) || {}
+  const presets = Array.isArray(cfg.presets) ? cfg.presets : []
+  const presetOptions = presets.length > 0
+    ? presets.map((p) => ({
+        value: String(p.name || p.id || ''),
+        label: `${p.name || p.id} (${(p.reference_models || p.references || []).length} models)`,
+      })).filter((o) => o.value !== '')
+    : [{ value: 'default', label: 'default' }]
+
+  const fields = [
+    {
+      path: ['enabled'],
+      group: 'general',
+      kind: 'boolean',
+      label: 'Enable MoA (/moa slash command)',
+      hint: 'Master switch for Mixture of Agents turn routing.',
+      format: boolFormat,
+      parse: boolParse,
+    },
+    {
+      path: ['default_preset'],
+      group: 'general',
+      kind: 'text',
+      label: 'Default preset',
+      hint: 'Active consensus preset used when /moa is invoked without --preset.',
+      options: presetOptions,
+      format: (value) => (typeof value === 'string' && value.trim() !== '' ? value.trim() : 'default'),
+      parse: (text) => {
+        const value = String(text).trim()
+        return value === '' ? { kind: 'set', value: 'default' } : { kind: 'set', value }
+      },
+    },
+    {
+      path: ['interactive_workbench_enabled'],
+      group: 'pipeline',
+      kind: 'boolean',
+      label: 'Interactive Workbench',
+      hint: 'Enable interactive per-file candidate assembly workbench.',
+      format: (val) => (val === true ? 'true' : 'false'),
+      parse: boolParse,
+    },
+    {
+      path: ['ask_clarifying_questions'],
+      group: 'pipeline',
+      kind: 'boolean',
+      label: 'Ask clarifying questions',
+      hint: 'Prompt user for clarification on broad or ambiguous requests before running ensemble.',
+      format: boolFormat,
+      parse: boolParse,
+    },
+    {
+      path: ['test_gate_enabled'],
+      group: 'pipeline',
+      kind: 'boolean',
+      label: 'Test Execution Gate',
+      hint: 'Run syntax and unit test checks against candidate solutions before judge ranking.',
+      format: (val) => (val === true ? 'true' : 'false'),
+      parse: boolParse,
+    },
+    {
+      path: ['smart_routing_enabled'],
+      group: 'advanced',
+      kind: 'boolean',
+      label: 'Smart Routing',
+      hint: 'Automatically select the best preset based on prompt complexity.',
+      format: (val) => (val === true ? 'true' : 'false'),
+      parse: boolParse,
+    },
+    {
+      path: ['uncertainty_scoring_enabled'],
+      group: 'advanced',
+      kind: 'boolean',
+      label: 'Token Uncertainty Scoring',
+      hint: 'Calculate token-level entropy from logprobs to highlight potential hallucinations for the judge.',
+      format: (val) => (val === true ? 'true' : 'false'),
+      parse: boolParse,
+    },
+  ]
+
+  return {
+    ns: 'dsh-moa',
+    title: 'Mixture of Agents (MoA)',
+    descriptions: {
+      en: 'Multi-model ensemble synthesis, judge consensus, and pipeline verification controls.',
+      zh: '多模型集成合成、裁判共识与流水线验证设置。',
+    },
+    groups: [
+      { id: 'general', title: 'General', descriptions: { zh: '常规' } },
+      { id: 'pipeline', title: 'Pipeline & Gates', descriptions: { zh: '流水线与验证' } },
+      { id: 'advanced', title: 'Advanced', descriptions: { zh: '高级' } },
+    ],
+    fields,
+  }
+}
+
+/**
+ * Registers the TUI settings section when the host provides tuiSettingsSections.
+ *
+ * @param {object} ctx
+ * @param {{ live: () => Record<string, any> }} deps
+ */
+export function applyTuiSettings(ctx, { live }) {
+  const registerWithService = (tuiCtx) => {
+    if (!tuiCtx) return
+    const tuiService = service(tuiCtx, 'tuiSettingsSections')
+    if (!tuiService || typeof tuiService.register !== 'function') return
+    try {
+      const dispose = tuiService.register(buildTuiSection({ live }))
+      const eff = typeof tuiCtx.effect === 'function' ? tuiCtx.effect : (typeof ctx?.effect === 'function' ? ctx.effect : null)
+      if (eff) {
+        eff(() => () => {
+          try { if (typeof dispose === 'function') dispose() } catch {}
+        }, 'dsh-moa: tui settings')
+      }
+    } catch (err) {
+      ctx?.logger?.warn?.('[dsh-moa] Could not register TUI settings: ' + (err?.message || err))
+    }
+  }
+
+  if (service(ctx, 'tuiSettingsSections')?.register) {
+    registerWithService(ctx)
+  } else if (typeof ctx?.inject === 'function') {
+    ctx.inject(['tuiSettingsSections'], registerWithService)
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-moa",
-  "version": "0.2.66",
+  "version": "0.2.67",
   "description": "Mixture of Agents (MoA) plugin for DeepSeek Harness with /moa slash command",
   "type": "module",
   "main": "lib/index.js",