@goodandready/dsh-moa 0.2.65 → 0.2.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +2000 -2245
- package/lib/index.js +16 -0
- package/lib/moa-candidates.js +1 -0
- package/lib/moa-model-validator.js +242 -0
- package/lib/moa-schema.js +5 -0
- package/lib/moa-service.js +21 -0
- package/lib/moa-uncertainty.js +154 -0
- package/lib/pipeline/synthesis.js +10 -1
- package/lib/routes/presets.js +11 -0
- package/lib/tui-settings.js +146 -0
- package/package.json +4 -2
package/lib/index.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { applyTuiSettings, buildTuiSection } from './tui-settings.js'
|
|
1
2
|
import { registerMoaCommands } from './commands.js'
|
|
2
3
|
import { service, getWebServerPort } from './services.js'
|
|
3
4
|
import crypto from 'node:crypto'
|
|
@@ -189,6 +190,9 @@ export function apply(ctx, config) {
|
|
|
189
190
|
// Official slash command /moa registration via dsh-commands (#275)
|
|
190
191
|
registerMoaCommands(ctx, { live, moaService })
|
|
191
192
|
|
|
193
|
+
// Official TUI settings section registration (#277)
|
|
194
|
+
applyTuiSettings(ctx, { live })
|
|
195
|
+
|
|
192
196
|
// Session turn interceptor: route turn to real aggregator model & stream MoA synthesis
|
|
193
197
|
ctx.effect(() => {
|
|
194
198
|
return () => {
|
|
@@ -385,3 +389,15 @@ export {
|
|
|
385
389
|
classifyLlmError,
|
|
386
390
|
isFailoverEligible,
|
|
387
391
|
} from './errors.js'
|
|
392
|
+
export {
|
|
393
|
+
validatePreset,
|
|
394
|
+
validateAllPresets,
|
|
395
|
+
getActiveLlmProviders,
|
|
396
|
+
getActiveLlmModels,
|
|
397
|
+
} from './moa-model-validator.js'
|
|
398
|
+
export { applyTuiSettings, buildTuiSection } from './tui-settings.js'
|
|
399
|
+
export {
|
|
400
|
+
calculateTokenEntropy,
|
|
401
|
+
analyzeCandidateUncertainty,
|
|
402
|
+
formatUncertaintyAnnotations,
|
|
403
|
+
} from './moa-uncertainty.js'
|
package/lib/moa-candidates.js
CHANGED
|
@@ -147,6 +147,7 @@ export async function runReferencesParallel(references, messages, options = {},
|
|
|
147
147
|
costUsd: costInfo.costUsd,
|
|
148
148
|
durationMs: Date.now() - candidateStart,
|
|
149
149
|
model: slot?.model ? (slot.provider ? (slot.provider + ":" + slot.model) : slot.model) : label,
|
|
150
|
+
...(res?.logprobs ? { logprobs: res.logprobs } : {}),
|
|
150
151
|
ok: true,
|
|
151
152
|
}
|
|
152
153
|
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dynamic Preset Model Validation via ctx.llm Provider Registry (#283)
|
|
3
|
+
* Inspects active providers and models, detects dead/unregistered slots,
|
|
4
|
+
* and produces diagnostic reports for logging and UI surfacing.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { logger } from './logger.js'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Discovers currently registered / active LLM providers in ctx.
|
|
11
|
+
* Queries ctx.llm.listProviders(), ctx.llm.listConfigurableProviders(),
|
|
12
|
+
* and active provider maps.
|
|
13
|
+
*
|
|
14
|
+
* @param {object} ctx
|
|
15
|
+
* @returns {Promise<Set<string>>}
|
|
16
|
+
*/
|
|
17
|
+
export async function getActiveLlmProviders(ctx) {
|
|
18
|
+
const providers = new Set()
|
|
19
|
+
if (!ctx || !ctx.llm) return providers
|
|
20
|
+
|
|
21
|
+
// 1. ctx.llm.listProviders()
|
|
22
|
+
if (typeof ctx.llm.listProviders === 'function') {
|
|
23
|
+
try {
|
|
24
|
+
const list = await ctx.llm.listProviders()
|
|
25
|
+
if (Array.isArray(list)) {
|
|
26
|
+
for (const p of list) {
|
|
27
|
+
const id = typeof p === 'string' ? p : (p?.id || p?.provider)
|
|
28
|
+
if (id) providers.add(String(id).toLowerCase())
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
} catch (_e) {}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// 2. ctx.llm.listConfigurableProviders()
|
|
35
|
+
if (typeof ctx.llm.listConfigurableProviders === 'function') {
|
|
36
|
+
try {
|
|
37
|
+
const list = ctx.llm.listConfigurableProviders()
|
|
38
|
+
if (Array.isArray(list)) {
|
|
39
|
+
for (const p of list) {
|
|
40
|
+
const id = typeof p === 'string' ? p : (p?.id || p?.provider)
|
|
41
|
+
if (id) providers.add(String(id).toLowerCase())
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
} catch (_e) {}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// 3. Fallback to ctx.llm.providers object or map
|
|
48
|
+
if (ctx.llm.providers) {
|
|
49
|
+
if (typeof ctx.llm.providers.keys === 'function') {
|
|
50
|
+
for (const k of ctx.llm.providers.keys()) providers.add(String(k).toLowerCase())
|
|
51
|
+
} else if (typeof ctx.llm.providers === 'object') {
|
|
52
|
+
for (const k of Object.keys(ctx.llm.providers)) providers.add(String(k).toLowerCase())
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
return providers
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Discovers registered models per provider if available.
|
|
61
|
+
*
|
|
62
|
+
* @param {object} ctx
|
|
63
|
+
* @param {Set<string>} [providers]
|
|
64
|
+
* @returns {Promise<Map<string, Set<string>>>} Map of provider -> Set of model IDs
|
|
65
|
+
*/
|
|
66
|
+
export async function getActiveLlmModels(ctx, providers) {
|
|
67
|
+
const modelMap = new Map()
|
|
68
|
+
if (!ctx || !ctx.llm) return modelMap
|
|
69
|
+
|
|
70
|
+
const targetProviders = providers || await getActiveLlmProviders(ctx)
|
|
71
|
+
|
|
72
|
+
// 1. ctx.llm.listAvailableModels()
|
|
73
|
+
if (typeof ctx.llm.listAvailableModels === 'function') {
|
|
74
|
+
try {
|
|
75
|
+
const all = await ctx.llm.listAvailableModels()
|
|
76
|
+
if (Array.isArray(all)) {
|
|
77
|
+
for (const item of all) {
|
|
78
|
+
const p = String(item?.provider || '').toLowerCase()
|
|
79
|
+
const m = String(item?.id || item?.model || '').toLowerCase()
|
|
80
|
+
if (p && m) {
|
|
81
|
+
if (!modelMap.has(p)) modelMap.set(p, new Set())
|
|
82
|
+
modelMap.get(p).add(m)
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
} catch (_e) {}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// 2. Query each provider with ctx.llm.listModels(p)
|
|
90
|
+
if (typeof ctx.llm.listModels === 'function') {
|
|
91
|
+
for (const p of targetProviders) {
|
|
92
|
+
try {
|
|
93
|
+
const mList = await Promise.race([
|
|
94
|
+
ctx.llm.listModels(p),
|
|
95
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 1500)),
|
|
96
|
+
])
|
|
97
|
+
if (Array.isArray(mList)) {
|
|
98
|
+
if (!modelMap.has(p)) modelMap.set(p, new Set())
|
|
99
|
+
for (const m of mList) {
|
|
100
|
+
const mId = typeof m === 'string' ? m : (m?.id || m?.model || m?.name)
|
|
101
|
+
if (mId) modelMap.get(p).add(String(mId).toLowerCase())
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
} catch (_e) {}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return modelMap
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Validates a single slot against active providers and models.
|
|
113
|
+
*/
|
|
114
|
+
export function validateSlot(slot, role, activeProviders, modelMap) {
|
|
115
|
+
if (!slot) return { valid: false, role, reason: 'empty_slot' }
|
|
116
|
+
const provider = String(slot.provider || '').toLowerCase().trim()
|
|
117
|
+
const model = String(slot.model || '').toLowerCase().trim()
|
|
118
|
+
const label = `${slot.provider || 'unknown'}:${slot.model || 'unknown'}`
|
|
119
|
+
|
|
120
|
+
if (!provider) {
|
|
121
|
+
return { valid: false, role, provider: slot.provider, model: slot.model, label, reason: 'missing_provider' }
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// If we have active providers discovered and provider is not in the list
|
|
125
|
+
if (activeProviders && activeProviders.size > 0 && !activeProviders.has(provider)) {
|
|
126
|
+
return { valid: false, role, provider: slot.provider, model: slot.model, label, reason: 'provider_not_found' }
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// If we have models list for this provider, check if model exists
|
|
130
|
+
if (modelMap && modelMap.has(provider)) {
|
|
131
|
+
const models = modelMap.get(provider)
|
|
132
|
+
if (models && models.size > 0 && !models.has(model)) {
|
|
133
|
+
return { valid: false, role, provider: slot.provider, model: slot.model, label, reason: 'model_not_found' }
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
return { valid: true, role, provider: slot.provider, model: slot.model, label }
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Validates a preset's reference models, aggregator, and fallbacks.
|
|
142
|
+
*
|
|
143
|
+
* @param {object} preset
|
|
144
|
+
* @param {object} [options]
|
|
145
|
+
* @returns {object}
|
|
146
|
+
*/
|
|
147
|
+
export function validatePreset(preset, { activeProviders, modelMap } = {}) {
|
|
148
|
+
if (!preset || typeof preset !== 'object') {
|
|
149
|
+
return {
|
|
150
|
+
valid: false,
|
|
151
|
+
presetName: 'unknown',
|
|
152
|
+
deadModels: [{ role: 'preset', reason: 'invalid_preset_object' }],
|
|
153
|
+
warnings: ['Preset is not a valid object'],
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const presetName = preset.name || preset.id || 'unnamed'
|
|
158
|
+
const deadModels = []
|
|
159
|
+
const warnings = []
|
|
160
|
+
|
|
161
|
+
// Check aggregator
|
|
162
|
+
const aggSlot = preset.aggregator_model || preset.aggregator
|
|
163
|
+
if (aggSlot) {
|
|
164
|
+
const res = validateSlot(aggSlot, 'aggregator', activeProviders, modelMap)
|
|
165
|
+
if (!res.valid) {
|
|
166
|
+
deadModels.push(res)
|
|
167
|
+
warnings.push(`Aggregator model ${res.label} has unreachable provider or model (${res.reason})`)
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Check reference models
|
|
172
|
+
const refs = Array.isArray(preset.references)
|
|
173
|
+
? preset.references
|
|
174
|
+
: (Array.isArray(preset.reference_models) ? preset.reference_models : [])
|
|
175
|
+
|
|
176
|
+
if (refs.length === 0) {
|
|
177
|
+
warnings.push('Preset has no reference models configured')
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
for (let idx = 0; idx < refs.length; idx++) {
|
|
181
|
+
const slot = refs[idx]
|
|
182
|
+
const res = validateSlot(slot, `reference_${idx + 1}`, activeProviders, modelMap)
|
|
183
|
+
if (!res.valid) {
|
|
184
|
+
deadModels.push(res)
|
|
185
|
+
warnings.push(`Reference candidate ${res.label} has unreachable provider or model (${res.reason})`)
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Check slot fallbacks if any
|
|
189
|
+
if (Array.isArray(slot?.fallbacks)) {
|
|
190
|
+
for (const fb of slot.fallbacks) {
|
|
191
|
+
const fbRes = validateSlot(fb, `fallback_for_${res.label}`, activeProviders, modelMap)
|
|
192
|
+
if (!fbRes.valid) {
|
|
193
|
+
deadModels.push(fbRes)
|
|
194
|
+
warnings.push(`Reserve fallback ${fbRes.label} for ${res.label} is unreachable (${fbRes.reason})`)
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
return {
|
|
201
|
+
presetId: preset.id || presetName,
|
|
202
|
+
presetName,
|
|
203
|
+
valid: deadModels.length === 0,
|
|
204
|
+
hasDeadModels: deadModels.length > 0,
|
|
205
|
+
deadModels,
|
|
206
|
+
warnings,
|
|
207
|
+
totalSlotsChecked: (aggSlot ? 1 : 0) + refs.length,
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Validates an array of presets and logs warnings if any dead models exist.
|
|
213
|
+
*
|
|
214
|
+
* @param {Array<object>} presets
|
|
215
|
+
* @param {object} ctx
|
|
216
|
+
* @returns {Promise<object>}
|
|
217
|
+
*/
|
|
218
|
+
export async function validateAllPresets(presets, ctx) {
|
|
219
|
+
const activeProviders = await getActiveLlmProviders(ctx)
|
|
220
|
+
const modelMap = await getActiveLlmModels(ctx, activeProviders)
|
|
221
|
+
|
|
222
|
+
const presetResults = []
|
|
223
|
+
let totalDeadModels = 0
|
|
224
|
+
|
|
225
|
+
for (const preset of (presets || [])) {
|
|
226
|
+
const result = validatePreset(preset, { activeProviders, modelMap })
|
|
227
|
+
presetResults.push(result)
|
|
228
|
+
if (result.hasDeadModels) {
|
|
229
|
+
totalDeadModels += result.deadModels.length
|
|
230
|
+
const deadLabels = result.deadModels.map((m) => `${m.label} (${m.reason})`).join(', ')
|
|
231
|
+
logger.warn(`[dsh-moa] Preset "${result.presetName}" contains ${result.deadModels.length} dead/unregistered models: ${deadLabels}`)
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
return {
|
|
236
|
+
valid: totalDeadModels === 0,
|
|
237
|
+
activeProviders: Array.from(activeProviders),
|
|
238
|
+
totalDeadModels,
|
|
239
|
+
presets: presetResults,
|
|
240
|
+
validatedAt: Date.now(),
|
|
241
|
+
}
|
|
242
|
+
}
|
package/lib/moa-schema.js
CHANGED
|
@@ -63,12 +63,17 @@ export const PresetSchema = z.object({
|
|
|
63
63
|
local_fallback_enabled: z.boolean().default(false),
|
|
64
64
|
local_fallback_models: z.array(ModelSlotSchema).default([]),
|
|
65
65
|
interactive_workbench_enabled: z.boolean().default(false),
|
|
66
|
+
uncertainty_scoring_enabled: z.boolean().default(false),
|
|
66
67
|
})
|
|
67
68
|
|
|
68
69
|
export const Config = z.object({
|
|
69
70
|
enabled: z.boolean().default(true).volatile(),
|
|
70
71
|
default_preset: z.string().default('default').volatile(),
|
|
71
72
|
smart_routing_enabled: z.boolean().default(false).volatile(),
|
|
73
|
+
interactive_workbench_enabled: z.boolean().default(false).volatile(),
|
|
74
|
+
ask_clarifying_questions: z.boolean().default(true).volatile(),
|
|
75
|
+
test_gate_enabled: z.boolean().default(false).volatile(),
|
|
76
|
+
uncertainty_scoring_enabled: z.boolean().default(false).volatile(),
|
|
72
77
|
smart_routing_model: ModelSlotSchema.default({ provider: '', model: '' }).volatile(),
|
|
73
78
|
prices: z.dict(PriceRow).default({}).volatile(),
|
|
74
79
|
presets: z.array(PresetSchema).default([
|
package/lib/moa-service.js
CHANGED
|
@@ -152,4 +152,25 @@ export class MoaService {
|
|
|
152
152
|
emit(event, payload) {
|
|
153
153
|
emitEvent(this.ctx, event, payload)
|
|
154
154
|
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Validate presets against active LLM providers in ctx.llm
|
|
158
|
+
* @param {Array<object>} [presets]
|
|
159
|
+
* @returns {Promise<object>}
|
|
160
|
+
*/
|
|
161
|
+
async validatePresets(presets) {
|
|
162
|
+
const list = presets || this.getPresets()
|
|
163
|
+
const { validateAllPresets } = await import('./moa-model-validator.js')
|
|
164
|
+
return validateAllPresets(list, this.ctx)
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Get list of currently active LLM provider names
|
|
169
|
+
* @returns {Promise<Array<string>>}
|
|
170
|
+
*/
|
|
171
|
+
async getActiveProviders() {
|
|
172
|
+
const { getActiveLlmProviders } = await import('./moa-model-validator.js')
|
|
173
|
+
const set = await getActiveLlmProviders(this.ctx)
|
|
174
|
+
return Array.from(set)
|
|
175
|
+
}
|
|
155
176
|
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token-Level Uncertainty & Entropy Scoring for Hallucination Detection (#256)
|
|
3
|
+
* Calculates Shannon entropy across token logprob distributions, detects high-uncertainty
|
|
4
|
+
* identifier/method call spans, and builds diagnostic annotations for the Judge.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const LN2 = Math.LN2
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Computes token entropy in bits from a token logprob entry.
|
|
11
|
+
* Supports full top_logprobs distributions as well as scalar logprob surprisal.
|
|
12
|
+
*
|
|
13
|
+
* @param {object|number} entry
|
|
14
|
+
* @returns {number} Entropy in bits (>= 0)
|
|
15
|
+
*/
|
|
16
|
+
export function calculateTokenEntropy(entry) {
|
|
17
|
+
if (typeof entry === 'number') {
|
|
18
|
+
if (!Number.isFinite(entry) || entry >= 0) return 0
|
|
19
|
+
return Number((-entry / LN2).toFixed(4))
|
|
20
|
+
}
|
|
21
|
+
if (!entry || typeof entry !== 'object') return 0
|
|
22
|
+
|
|
23
|
+
const topList = Array.isArray(entry.top_logprobs) ? entry.top_logprobs : null
|
|
24
|
+
if (topList && topList.length > 0) {
|
|
25
|
+
let entropy = 0
|
|
26
|
+
let totalProb = 0
|
|
27
|
+
for (const item of topList) {
|
|
28
|
+
const lp = typeof item === 'number' ? item : Number(item?.logprob)
|
|
29
|
+
if (!Number.isFinite(lp)) continue
|
|
30
|
+
const p = Math.min(1, Math.max(1e-9, Math.exp(lp)))
|
|
31
|
+
totalProb += p
|
|
32
|
+
entropy -= p * (lp / LN2)
|
|
33
|
+
}
|
|
34
|
+
// Account for remaining probability mass if top_logprobs is truncated
|
|
35
|
+
if (totalProb < 0.999 && totalProb > 0) {
|
|
36
|
+
const rem = 1 - totalProb
|
|
37
|
+
entropy -= rem * Math.log2(rem)
|
|
38
|
+
}
|
|
39
|
+
return Number(Math.max(0, entropy).toFixed(4))
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const lp = Number(entry.logprob)
|
|
43
|
+
if (Number.isFinite(lp) && lp < 0) {
|
|
44
|
+
return Number((-lp / LN2).toFixed(4))
|
|
45
|
+
}
|
|
46
|
+
return 0
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Analyzes a candidate output's token logprobs (and code heuristics) to identify
|
|
51
|
+
* high-uncertainty zones where hallucinations are likely.
|
|
52
|
+
*
|
|
53
|
+
* @param {object} candidate
|
|
54
|
+
* @param {object} [options]
|
|
55
|
+
* @returns {object}
|
|
56
|
+
*/
|
|
57
|
+
export function analyzeCandidateUncertainty(candidate, options = {}) {
|
|
58
|
+
const threshold = typeof options.entropyThreshold === 'number' ? options.entropyThreshold : 1.2
|
|
59
|
+
const rawLogprobs = Array.isArray(candidate?.logprobs)
|
|
60
|
+
? candidate.logprobs
|
|
61
|
+
: (Array.isArray(candidate?.logprobs?.content) ? candidate.logprobs.content : [])
|
|
62
|
+
|
|
63
|
+
const highEntropySpans = []
|
|
64
|
+
let sumEntropy = 0
|
|
65
|
+
let maxEntropy = 0
|
|
66
|
+
let tokenCount = 0
|
|
67
|
+
|
|
68
|
+
if (rawLogprobs.length > 0) {
|
|
69
|
+
for (let i = 0; i < rawLogprobs.length; i++) {
|
|
70
|
+
const item = rawLogprobs[i]
|
|
71
|
+
const h = calculateTokenEntropy(item)
|
|
72
|
+
const tok = String(item?.token ?? '').trim()
|
|
73
|
+
sumEntropy += h
|
|
74
|
+
if (h > maxEntropy) maxEntropy = h
|
|
75
|
+
tokenCount++
|
|
76
|
+
|
|
77
|
+
if (h >= threshold && tok.length > 0 && !/^[\s{}()\[\];,.:'"`]+$/.test(tok)) {
|
|
78
|
+
// Reconstruct local context window around token
|
|
79
|
+
const prev = rawLogprobs.slice(Math.max(0, i - 2), i).map((t) => t?.token || '').join('')
|
|
80
|
+
const next = rawLogprobs.slice(i + 1, Math.min(rawLogprobs.length, i + 3)).map((t) => t?.token || '').join('')
|
|
81
|
+
const contextSpan = (prev + (item?.token || '') + next).replace(/\s+/g, ' ').trim()
|
|
82
|
+
const prob = Number.isFinite(Number(item?.logprob))
|
|
83
|
+
? Number(Math.exp(Number(item.logprob)).toFixed(3))
|
|
84
|
+
: Number(Math.pow(2, -h).toFixed(3))
|
|
85
|
+
|
|
86
|
+
highEntropySpans.push({
|
|
87
|
+
token: tok,
|
|
88
|
+
span: contextSpan || tok,
|
|
89
|
+
entropy: h,
|
|
90
|
+
probability: prob,
|
|
91
|
+
warning: `Внимание: высокая энтропия на участке "${contextSpan || tok}" (H=${h.toFixed(2)} bits, p=${prob})`,
|
|
92
|
+
})
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const avgEntropy = tokenCount > 0 ? Number((sumEntropy / tokenCount).toFixed(4)) : 0
|
|
98
|
+
let riskLevel = 'low'
|
|
99
|
+
if (maxEntropy >= threshold * 1.6 || highEntropySpans.length >= 3) {
|
|
100
|
+
riskLevel = 'high'
|
|
101
|
+
} else if (maxEntropy >= threshold || highEntropySpans.length >= 1) {
|
|
102
|
+
riskLevel = 'medium'
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return {
|
|
106
|
+
candidateIndex: candidate?.index ?? 0,
|
|
107
|
+
model: candidate?.label || candidate?.model || 'unknown',
|
|
108
|
+
tokenCount,
|
|
109
|
+
avgEntropy,
|
|
110
|
+
maxEntropy: Number(maxEntropy.toFixed(4)),
|
|
111
|
+
riskLevel,
|
|
112
|
+
highEntropySpans,
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Formats uncertainty analysis across all candidates into a Judge prompt directive.
|
|
118
|
+
*
|
|
119
|
+
* @param {Array<object>} candidates
|
|
120
|
+
* @param {object} [options]
|
|
121
|
+
* @returns {{ reportText: string, analyses: Array<object>, hasHighUncertainty: boolean }}
|
|
122
|
+
*/
|
|
123
|
+
export function formatUncertaintyAnnotations(candidates, options = {}) {
|
|
124
|
+
if (!Array.isArray(candidates) || candidates.length === 0) {
|
|
125
|
+
return { reportText: '', analyses: [], hasHighUncertainty: false }
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const analyses = candidates.map((c, idx) =>
|
|
129
|
+
analyzeCandidateUncertainty({ ...c, index: c.index ?? idx + 1 }, options)
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
const flagged = analyses.filter((a) => a.highEntropySpans.length > 0)
|
|
133
|
+
if (flagged.length === 0) {
|
|
134
|
+
return { reportText: '', analyses, hasHighUncertainty: false }
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const lines = [
|
|
138
|
+
'\n### 🔬 Token-Level Uncertainty & Hallucination Warnings:',
|
|
139
|
+
'The following candidate segments exhibited abnormally high token entropy (low model confidence). Carefully verify API existence, method signatures, and edge cases before adopting these fragments:',
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
for (const a of flagged) {
|
|
143
|
+
lines.push(`- **Candidate ${a.candidateIndex} (${a.model})** [Risk: ${a.riskLevel.toUpperCase()}, Avg H=${a.avgEntropy}, Max H=${a.maxEntropy}]:`)
|
|
144
|
+
for (const span of a.highEntropySpans.slice(0, 5)) {
|
|
145
|
+
lines.push(` - ⚠️ ${span.warning}`)
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return {
|
|
150
|
+
reportText: lines.join('\n') + '\n',
|
|
151
|
+
analyses,
|
|
152
|
+
hasHighUncertainty: true,
|
|
153
|
+
}
|
|
154
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { formatUncertaintyAnnotations } from '../moa-uncertainty.js'
|
|
1
2
|
import { logger } from '../logger.js'
|
|
2
3
|
import { buildSynthesisPrompt, slotLabel } from '../moa-prompts.js'
|
|
3
4
|
import { estimateDownstreamCost, checkPhaseBudget, createBudgetAbortPayload } from '../moa-budget.js'
|
|
@@ -36,7 +37,14 @@ export async function stageSynthesis({
|
|
|
36
37
|
collectedCtx,
|
|
37
38
|
startTime,
|
|
38
39
|
}) {
|
|
39
|
-
|
|
40
|
+
let uncertaintyDirective = ''
|
|
41
|
+
if (preset?.uncertainty_scoring_enabled) {
|
|
42
|
+
const uncReport = formatUncertaintyAnnotations(referenceOutputs)
|
|
43
|
+
if (uncReport.hasHighUncertainty) {
|
|
44
|
+
uncertaintyDirective = uncReport.reportText
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const baseSynthesisPrompt = buildSynthesisPrompt(userPrompt, referenceOutputs, judgeCriteria, {
|
|
40
48
|
curatorSynthesis: isCuratorSynthesis,
|
|
41
49
|
blindEvaluation: isBlindEvaluation,
|
|
42
50
|
priorTurnBaseline,
|
|
@@ -44,6 +52,7 @@ export async function stageSynthesis({
|
|
|
44
52
|
fileScoringDirective,
|
|
45
53
|
consensusReport: multiJudgeResult?.consensusReport,
|
|
46
54
|
})
|
|
55
|
+
const synthesisPrompt = uncertaintyDirective ? baseSynthesisPrompt + '\n' + uncertaintyDirective : baseSynthesisPrompt
|
|
47
56
|
const synthPromptTokens = Math.max(50, Math.ceil(synthesisPrompt.length / 4))
|
|
48
57
|
|
|
49
58
|
if (preset?.budget_guard_enabled && preset?.max_budget_usd > 0 && !isFastMode) {
|
package/lib/routes/presets.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { validateAllPresets } from '../moa-model-validator.js'
|
|
1
2
|
import { writeJson, readBody, assertAuthorizedRequest } from '../access.js'
|
|
2
3
|
import { isSafeWriteRequest } from '../updater.js'
|
|
3
4
|
import { resolvePresetForPrompt } from '../moa-router.js'
|
|
@@ -134,10 +135,15 @@ ctx.effect(() => {
|
|
|
134
135
|
if (!assertAuthorizedRequest(req, res, ctx)) return
|
|
135
136
|
const cfg = live()
|
|
136
137
|
if (req.method === 'GET') {
|
|
138
|
+
let validation = null
|
|
139
|
+
try {
|
|
140
|
+
validation = await validateAllPresets(cfg.presets || [], ctx)
|
|
141
|
+
} catch (_e) {}
|
|
137
142
|
writeJson(res, 200, {
|
|
138
143
|
ok: true,
|
|
139
144
|
defaultPreset: cfg.default_preset || 'default',
|
|
140
145
|
presets: cfg.presets || [],
|
|
146
|
+
validation,
|
|
141
147
|
})
|
|
142
148
|
return
|
|
143
149
|
}
|
|
@@ -179,12 +185,17 @@ ctx.effect(() => {
|
|
|
179
185
|
}
|
|
180
186
|
|
|
181
187
|
const unwrapped = typeof plainConfig === 'function' ? plainConfig(validated) : validated
|
|
188
|
+
let validation = null
|
|
189
|
+
try {
|
|
190
|
+
validation = await validateAllPresets(unwrapped.presets || [], ctx)
|
|
191
|
+
} catch (_e) {}
|
|
182
192
|
writeJson(res, 200, {
|
|
183
193
|
ok: true,
|
|
184
194
|
persisted: persistResult?.persisted ?? false,
|
|
185
195
|
enabled: unwrapped.enabled !== false,
|
|
186
196
|
defaultPreset: unwrapped.default_preset,
|
|
187
197
|
presets: unwrapped.presets,
|
|
198
|
+
validation,
|
|
188
199
|
})
|
|
189
200
|
} catch (err) { writeJson(res, 400, { ok: false, error: err?.message || String(err) }) }
|
|
190
201
|
return
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DSH Terminal User Interface (TUI) Settings Section for dsh-moa (#277)
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { service } from './services.js'
|
|
6
|
+
|
|
7
|
+
function boolFormat(value) {
|
|
8
|
+
return value === false ? 'false' : 'true'
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function boolParse(text) {
|
|
12
|
+
return { kind: 'set', value: String(text).trim() === 'true' }
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Builds the TUI settings section descriptor from live configuration.
|
|
17
|
+
*
|
|
18
|
+
* @param {{ live: () => Record<string, any> }} deps
|
|
19
|
+
* @returns {object}
|
|
20
|
+
*/
|
|
21
|
+
export function buildTuiSection({ live }) {
|
|
22
|
+
const cfg = (typeof live === 'function' ? live() : live) || {}
|
|
23
|
+
const presets = Array.isArray(cfg.presets) ? cfg.presets : []
|
|
24
|
+
const presetOptions = presets.length > 0
|
|
25
|
+
? presets.map((p) => ({
|
|
26
|
+
value: String(p.name || p.id || ''),
|
|
27
|
+
label: `${p.name || p.id} (${(p.reference_models || p.references || []).length} models)`,
|
|
28
|
+
})).filter((o) => o.value !== '')
|
|
29
|
+
: [{ value: 'default', label: 'default' }]
|
|
30
|
+
|
|
31
|
+
const fields = [
|
|
32
|
+
{
|
|
33
|
+
path: ['enabled'],
|
|
34
|
+
group: 'general',
|
|
35
|
+
kind: 'boolean',
|
|
36
|
+
label: 'Enable MoA (/moa slash command)',
|
|
37
|
+
hint: 'Master switch for Mixture of Agents turn routing.',
|
|
38
|
+
format: boolFormat,
|
|
39
|
+
parse: boolParse,
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
path: ['default_preset'],
|
|
43
|
+
group: 'general',
|
|
44
|
+
kind: 'text',
|
|
45
|
+
label: 'Default preset',
|
|
46
|
+
hint: 'Active consensus preset used when /moa is invoked without --preset.',
|
|
47
|
+
options: presetOptions,
|
|
48
|
+
format: (value) => (typeof value === 'string' && value.trim() !== '' ? value.trim() : 'default'),
|
|
49
|
+
parse: (text) => {
|
|
50
|
+
const value = String(text).trim()
|
|
51
|
+
return value === '' ? { kind: 'set', value: 'default' } : { kind: 'set', value }
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
path: ['interactive_workbench_enabled'],
|
|
56
|
+
group: 'pipeline',
|
|
57
|
+
kind: 'boolean',
|
|
58
|
+
label: 'Interactive Workbench',
|
|
59
|
+
hint: 'Enable interactive per-file candidate assembly workbench.',
|
|
60
|
+
format: (val) => (val === true ? 'true' : 'false'),
|
|
61
|
+
parse: boolParse,
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
path: ['ask_clarifying_questions'],
|
|
65
|
+
group: 'pipeline',
|
|
66
|
+
kind: 'boolean',
|
|
67
|
+
label: 'Ask clarifying questions',
|
|
68
|
+
hint: 'Prompt user for clarification on broad or ambiguous requests before running ensemble.',
|
|
69
|
+
format: boolFormat,
|
|
70
|
+
parse: boolParse,
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
path: ['test_gate_enabled'],
|
|
74
|
+
group: 'pipeline',
|
|
75
|
+
kind: 'boolean',
|
|
76
|
+
label: 'Test Execution Gate',
|
|
77
|
+
hint: 'Run syntax and unit test checks against candidate solutions before judge ranking.',
|
|
78
|
+
format: (val) => (val === true ? 'true' : 'false'),
|
|
79
|
+
parse: boolParse,
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
path: ['smart_routing_enabled'],
|
|
83
|
+
group: 'advanced',
|
|
84
|
+
kind: 'boolean',
|
|
85
|
+
label: 'Smart Routing',
|
|
86
|
+
hint: 'Automatically select the best preset based on prompt complexity.',
|
|
87
|
+
format: (val) => (val === true ? 'true' : 'false'),
|
|
88
|
+
parse: boolParse,
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
path: ['uncertainty_scoring_enabled'],
|
|
92
|
+
group: 'advanced',
|
|
93
|
+
kind: 'boolean',
|
|
94
|
+
label: 'Token Uncertainty Scoring',
|
|
95
|
+
hint: 'Calculate token-level entropy from logprobs to highlight potential hallucinations for the judge.',
|
|
96
|
+
format: (val) => (val === true ? 'true' : 'false'),
|
|
97
|
+
parse: boolParse,
|
|
98
|
+
},
|
|
99
|
+
]
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
ns: 'dsh-moa',
|
|
103
|
+
title: 'Mixture of Agents (MoA)',
|
|
104
|
+
descriptions: {
|
|
105
|
+
en: 'Multi-model ensemble synthesis, judge consensus, and pipeline verification controls.',
|
|
106
|
+
zh: '多模型集成合成、裁判共识与流水线验证设置。',
|
|
107
|
+
},
|
|
108
|
+
groups: [
|
|
109
|
+
{ id: 'general', title: 'General', descriptions: { zh: '常规' } },
|
|
110
|
+
{ id: 'pipeline', title: 'Pipeline & Gates', descriptions: { zh: '流水线与验证' } },
|
|
111
|
+
{ id: 'advanced', title: 'Advanced', descriptions: { zh: '高级' } },
|
|
112
|
+
],
|
|
113
|
+
fields,
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Registers the TUI settings section when the host provides tuiSettingsSections.
|
|
119
|
+
*
|
|
120
|
+
* @param {object} ctx
|
|
121
|
+
* @param {{ live: () => Record<string, any> }} deps
|
|
122
|
+
*/
|
|
123
|
+
export function applyTuiSettings(ctx, { live }) {
|
|
124
|
+
const registerWithService = (tuiCtx) => {
|
|
125
|
+
if (!tuiCtx) return
|
|
126
|
+
const tuiService = service(tuiCtx, 'tuiSettingsSections')
|
|
127
|
+
if (!tuiService || typeof tuiService.register !== 'function') return
|
|
128
|
+
try {
|
|
129
|
+
const dispose = tuiService.register(buildTuiSection({ live }))
|
|
130
|
+
const eff = typeof tuiCtx.effect === 'function' ? tuiCtx.effect : (typeof ctx?.effect === 'function' ? ctx.effect : null)
|
|
131
|
+
if (eff) {
|
|
132
|
+
eff(() => () => {
|
|
133
|
+
try { if (typeof dispose === 'function') dispose() } catch {}
|
|
134
|
+
}, 'dsh-moa: tui settings')
|
|
135
|
+
}
|
|
136
|
+
} catch (err) {
|
|
137
|
+
ctx?.logger?.warn?.('[dsh-moa] Could not register TUI settings: ' + (err?.message || err))
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
if (service(ctx, 'tuiSettingsSections')?.register) {
|
|
142
|
+
registerWithService(ctx)
|
|
143
|
+
} else if (typeof ctx?.inject === 'function') {
|
|
144
|
+
ctx.inject(['tuiSettingsSections'], registerWithService)
|
|
145
|
+
}
|
|
146
|
+
}
|