@goodandready/dsh-moa 0.2.64 → 0.2.66

package/lib/commands.js

@@ -0,0 +1,83 @@
+/**
+ * Official Slash Command Registration for /moa via dsh-commands service (#275)
+ */
+
+import { service } from './services.js'
+
+export function registerMoaCommands(ctx, { live, moaService }) {
+  const registerWithService = (cmdCtx) => {
+    if (!cmdCtx) return
+    const commands = service(cmdCtx, 'commands')
+    if (!commands || typeof commands.register !== 'function') return
+
+    const formatPresetsList = (cfg) => {
+      const presets = Array.isArray(cfg?.presets) ? cfg.presets : []
+      if (presets.length === 0) return 'No presets configured in dsh-moa.'
+      return '### 🧠 Configured MoA Presets:\n' + presets.map((p, i) =>
+        `- **${p.name}** (${p.id || `preset-${i + 1}`}): ${p.description || 'Consensus preset'} | ${p.references?.length || p.reference_models?.length || 0} advisors`
+      ).join('\n')
+    }
+
+    const commandHandler = async (invocation) => {
+      const rawText = typeof invocation === 'string' ? invocation : (invocation?.text || invocation?.raw || invocation?.args || '')
+      const trimmed = String(rawText).trim()
+
+      if (!trimmed || trimmed === 'help' || trimmed === '--help') {
+        return {
+          text: '🧠 **MoA Command Guide**:\n- `/moa presets` — list configured presets\n- `/moa history` — view recent runs\n- `/moa status` — check plugin status\n- `/moa [--preset <name>] <prompt>` — run MoA consensus in chat',
+        }
+      }
+
+      if (trimmed === 'presets' || trimmed === 'list') {
+        const cfg = typeof live === 'function' ? live() : {}
+        return { text: formatPresetsList(cfg) }
+      }
+
+      if (trimmed === 'status') {
+        const cfg = typeof live === 'function' ? live() : {}
+        return {
+          text: `🧠 **MoA Plugin Status**: ${cfg?.enabled !== false ? 'Enabled' : 'Disabled'} | Default preset: \`${cfg?.default_preset || 'default'}\``,
+        }
+      }
+
+      if (trimmed === 'history') {
+        const hist = moaService?.getHistory ? moaService.getHistory(5, 0) : { runs: [] }
+        if (!hist?.runs?.length) return { text: 'No MoA runs recorded in history.' }
+        return {
+          text: '### 📜 Recent MoA Runs:\n' + hist.runs.slice(0, 5).map((r) =>
+            `- \`${r.id?.slice(0, 8)}\` (${r.preset || 'default'}): "${r.prompt?.slice(0, 40)}..." (Winner: ${r.winnerModel || 'none'}, $${(r.totalCostUsd || 0).toFixed(4)})`
+          ).join('\n'),
+        }
+      }
+
+      return {
+        text: `🧠 MoA turn dispatched: "${trimmed}". The turn will execute with live consensus streaming.`,
+      }
+    }
+
+    const unregister = commands.register({
+      definitionId: '@goodandready/dsh-moa',
+      name: 'moa',
+      description: 'Run Mixture of Agents consensus or inspect presets (/moa [presets|history|status|<prompt>])',
+      input: { hint: '[presets|history|status|<prompt>]' },
+      handler: commandHandler,
+      execute: async (rawArgs, invocation = null) => {
+        const res = await commandHandler(invocation || rawArgs)
+        return res?.text || String(res || '')
+      },
+    })
+
+    const eff = typeof cmdCtx.effect === 'function' ? cmdCtx.effect : (fn) => fn()
+    eff(() => () => {
+      try {
+        if (typeof unregister === 'function') unregister()
+      } catch (_e) {}
+    }, 'dsh-moa: slash command registration')
+  }
+
+  if (service(ctx, 'commands')?.register) {
+    registerWithService(ctx)
+  } else if (typeof ctx.inject === 'function') {
+    ctx.inject(['commands'], registerWithService)
+  }
+}

package/lib/errors.js

@@ -0,0 +1,176 @@
+/**
+ * Centralized LLM and MoA Pipeline Error Taxonomy and Classification
+ */
+
+export class MoaError extends Error {
+  /**
+   * @param {string} message
+   * @param {object} [options]
+   */
+  constructor(message, options = {}) {
+    super(message)
+    this.name = new.target.name || 'MoaError'
+    this.code = options.code || 'MOA_ERROR'
+    this.status = typeof options.status === 'number' ? options.status : undefined
+    this.isRetryable = Boolean(options.isRetryable)
+    this.fallbackEligible = options.fallbackEligible !== false
+    this.cause = options.cause
+  }
+}
+
+export class RateLimitError extends MoaError {
+  constructor(message, options = {}) {
+    super(message, {
+      code: 'RATE_LIMIT_EXCEEDED',
+      status: 429,
+      isRetryable: true,
+      fallbackEligible: true,
+      ...options,
+    })
+  }
+}
+
+export class ContextOverflowError extends MoaError {
+  constructor(message, options = {}) {
+    super(message, {
+      code: 'CONTEXT_WINDOW_EXCEEDED',
+      status: 400,
+      isRetryable: false,
+      fallbackEligible: true,
+      ...options,
+    })
+  }
+}
+
+export class AuthenticationError extends MoaError {
+  constructor(message, options = {}) {
+    super(message, {
+      code: 'AUTHENTICATION_FAILED',
+      status: options.status || 401,
+      isRetryable: false,
+      fallbackEligible: false,
+      ...options,
+    })
+  }
+}
+
+export class ModelNotFoundError extends MoaError {
+  constructor(message, options = {}) {
+    super(message, {
+      code: 'MODEL_NOT_FOUND',
+      status: 404,
+      isRetryable: false,
+      fallbackEligible: true,
+      ...options,
+    })
+  }
+}
+
+export class TransientNetworkError extends MoaError {
+  constructor(message, options = {}) {
+    super(message, {
+      code: 'TRANSIENT_NETWORK_ERROR',
+      status: options.status || 503,
+      isRetryable: true,
+      fallbackEligible: true,
+      ...options,
+    })
+  }
+}
+
+/**
+ * Classifies an unknown error or exception into a structured MoA domain error.
+ *
+ * @param {unknown} err
+ * @returns {MoaError}
+ */
+export function classifyLlmError(err) {
+  if (!err) return new MoaError('Unknown error')
+  if (err instanceof MoaError) return err
+
+  const msg = String(err.message || err).toLowerCase()
+  const status = Number(err.status || err.statusCode || (msg.match(/\b(401|403|404|429|500|502|503|504)\b/)?.[1])) || undefined
+  const code = String(err.code || '').toUpperCase()
+
+  // Rate limit / Quota
+  if (
+    status === 429 ||
+    msg.includes('rate limit') ||
+    msg.includes('too many requests') ||
+    msg.includes('tpm') ||
+    msg.includes('rpm') ||
+    msg.includes('quota exceeded') ||
+    code.includes('RATE_LIMIT') ||
+    code.includes('QUOTA')
+  ) {
+    return new RateLimitError(err.message || String(err), { status: 429, cause: err })
+  }
+
+  // Context window overflow
+  if (
+    msg.includes('context window') ||
+    msg.includes('context length') ||
+    msg.includes('maximum context') ||
+    msg.includes('too long') ||
+    msg.includes('token limit') ||
+    code.includes('CONTEXT_WINDOW')
+  ) {
+    return new ContextOverflowError(err.message || String(err), { status: 400, cause: err })
+  }
+
+  // Authentication & permissions
+  if (
+    status === 401 ||
+    status === 403 ||
+    msg.includes('unauthorized') ||
+    msg.includes('invalid api key') ||
+    msg.includes('forbidden') ||
+    msg.includes('authentication') ||
+    code.includes('AUTH') ||
+    code.includes('INVALID_CREDENTIAL')
+  ) {
+    return new AuthenticationError(err.message || String(err), { status: status || 401, cause: err })
+  }
+
+  // Model not found / unavailable
+  if (
+    status === 404 ||
+    msg.includes('model not found') ||
+    msg.includes('does not exist') ||
+    msg.includes('unknown model') ||
+    code.includes('MODEL_NOT_FOUND')
+  ) {
+    return new ModelNotFoundError(err.message || String(err), { status: 404, cause: err })
+  }
+
+  // Transient network / gateway / timeout
+  if (
+    status === 502 ||
+    status === 503 ||
+    status === 504 ||
+    msg.includes('timeout') ||
+    msg.includes('econnreset') ||
+    msg.includes('etimedout') ||
+    msg.includes('network') ||
+    msg.includes('socket hang up') ||
+    msg.includes('service unavailable') ||
+    msg.includes('bad gateway') ||
+    code.includes('TIMEOUT') ||
+    code.includes('ECONNRESET')
+  ) {
+    return new TransientNetworkError(err.message || String(err), { status: status || 503, cause: err })
+  }
+
+  return new MoaError(err.message || String(err), { status, cause: err })
+}
+
+/**
+ * Returns true if the error qualifies for candidate failover to a reserve model.
+ *
+ * @param {unknown} err
+ * @returns {boolean}
+ */
+export function isFailoverEligible(err) {
+  const classified = classifyLlmError(err)
+  return classified.fallbackEligible
+}

package/lib/index.js

@@ -1,3 +1,4 @@
+import { registerMoaCommands } from './commands.js'
 import { service, getWebServerPort } from './services.js'
 import crypto from 'node:crypto'
 import path from 'node:path'
@@ -185,6 +186,9 @@ export function apply(ctx, config) {
     ctx.inject(['webServer'], registerWebSurfaces)
   }
 
+  // Official slash command /moa registration via dsh-commands (#275)
+  registerMoaCommands(ctx, { live, moaService })
+
   // Session turn interceptor: route turn to real aggregator model & stream MoA synthesis
   ctx.effect(() => {
     return () => {
@@ -370,3 +374,20 @@ export {
   handleWorkbenchGet,
   handleWorkbenchAssemble,
 } from './moa-workbench.js'
+export { registerMoaCommands } from './commands.js'
+export {
+  MoaError,
+  RateLimitError,
+  ContextOverflowError,
+  AuthenticationError,
+  ModelNotFoundError,
+  TransientNetworkError,
+  classifyLlmError,
+  isFailoverEligible,
+} from './errors.js'
+export {
+  validatePreset,
+  validateAllPresets,
+  getActiveLlmProviders,
+  getActiveLlmModels,
+} from './moa-model-validator.js'

package/lib/moa-candidates.js

@@ -1,3 +1,4 @@
+import { classifyLlmError, isFailoverEligible } from './errors.js'
 import { logger } from './logger.js'
 /**
  * DeepSeek Harness Mixture of Agents (MoA) — Candidate Fan-Out & Retry
@@ -18,14 +19,13 @@ export async function callWithTransientRetry(callLlmFn, callArgs, maxRetries = 0
       return await callLlmFn(callArgs)
     } catch (err) {
       attempt++
-      const msg = err?.message || String(err)
-      const isTransient = /429|rate limit|502|503|504|econnreset|etimedout|socket hang up/i.test(msg)
-      if (attempt <= maxRetries && isTransient && !callArgs?.signal?.aborted) {
+      const classified = classifyLlmError(err)
+      if (attempt <= maxRetries && classified.isRetryable && !callArgs?.signal?.aborted) {
         await new Promise((r) => setTimeout(r, retryDelayMs))
         if (callArgs?.signal?.aborted) throw new Error('Aborted')
         continue
       }
-      throw err
+      throw classified
     }
   }
 }
@@ -176,10 +176,29 @@ export async function runReferencesParallel(references, messages, options = {},
           return
         }
 
-        // Feature 10: Local fallback models if online candidate fails
+        const classified = classifyLlmError(err)
+        const canFailover = isFailoverEligible(classified)
+
+        // Reserve model failover: checks slot.fallbacks, options.fallback_models, or options.local_fallback_models
         let fallbackRecovered = false
-        if (options.local_fallback_enabled && Array.isArray(options.local_fallback_models) && options.local_fallback_models.length > 0) {
-          for (const fbSlot of options.local_fallback_models) {
+        const fallbackCandidates = [
+          ...(Array.isArray(slot?.fallbacks) ? slot.fallbacks : []),
+          ...(Array.isArray(options.fallback_models) ? options.fallback_models : []),
+          ...(options.local_fallback_enabled && Array.isArray(options.local_fallback_models) ? options.local_fallback_models : []),
+        ].filter(Boolean)
+
+        const seenFb = new Set([`${slot.provider}:${slot.model}`])
+        const eligibleFallbacks = []
+        for (const fb of fallbackCandidates) {
+          const key = `${fb.provider}:${fb.model}`
+          if (!seenFb.has(key)) {
+            seenFb.add(key)
+            eligibleFallbacks.push(fb)
+          }
+        }
+
+        if (canFailover && eligibleFallbacks.length > 0) {
+          for (const fbSlot of eligibleFallbacks) {
             if (options.signal?.aborted) break
             const fbLabel = slotLabel(fbSlot)
             const fbAbortCtrl = new AbortController()
@@ -198,7 +217,7 @@ export async function runReferencesParallel(references, messages, options = {},
 
             try {
               if (typeof onProgress === 'function') {
-                onProgress(`🔄 *Candidate ${i + 1} (${label}) failed, attempting local fallback to ${fbLabel}...*\n`)
+                onProgress(`🔄 *Candidate ${i + 1} (${label}) hit ${classified.name}, failover to ${fbLabel}...*\n`)
               }
               const fbCallPromise = callWithTransientRetry(
                 callLlm,
@@ -243,14 +262,15 @@ export async function runReferencesParallel(references, messages, options = {},
                 ok: true,
                 was_fallback: true,
                 original_model: label,
+                failoverReason: classified.code,
               }
               fallbackRecovered = true
               if (typeof onProgress === 'function') {
-                onProgress(`✅ *Candidate ${i + 1}/${total} recovered using fallback (${fbLabel})*\n`)
+                onProgress(`✅ *Candidate ${i + 1}/${total} recovered using failover (${fbLabel})*\n`)
               }
               break
             } catch (fbErr) {
-              logger.warn(`[dsh-moa] Fallback ${fbLabel} also failed:`, fbErr?.message || fbErr)
+              logger.warn(`[dsh-moa] Reserve model ${fbLabel} also failed:`, fbErr?.message || fbErr)
             } finally {
               if (fbTimer) clearTimeout(fbTimer)
               if (removeParentAbort) removeParentAbort()
@@ -260,7 +280,7 @@ export async function runReferencesParallel(references, messages, options = {},
 
         if (!fallbackRecovered) {
           finishedCount++
-          const errMsg = err?.message || String(err)
+          const errMsg = classified?.message || err?.message || String(err)
           logger.warn(`[dsh-moa] Reference ${label} failed:`, errMsg)
           if (typeof onProgress === 'function') {
             onProgress(`⚠️ *Candidate ${i + 1}/${total} (${label}) error: ${errMsg}*\n`)
@@ -276,6 +296,8 @@ export async function runReferencesParallel(references, messages, options = {},
             costUsd: 0,
             ok: false,
             error: errMsg,
+            errorClass: classified.name,
+            errorCode: classified.code,
           }
         }
       } finally {

package/lib/moa-model-validator.js

@@ -0,0 +1,242 @@
+/**
+ * Dynamic Preset Model Validation via ctx.llm Provider Registry (#283)
+ * Inspects active providers and models, detects dead/unregistered slots,
+ * and produces diagnostic reports for logging and UI surfacing.
+ */
+
+import { logger } from './logger.js'
+
+/**
+ * Discovers currently registered / active LLM providers in ctx.
+ * Queries ctx.llm.listProviders(), ctx.llm.listConfigurableProviders(),
+ * and active provider maps.
+ *
+ * @param {object} ctx
+ * @returns {Promise<Set<string>>}
+ */
+export async function getActiveLlmProviders(ctx) {
+  const providers = new Set()
+  if (!ctx || !ctx.llm) return providers
+
+  // 1. ctx.llm.listProviders()
+  if (typeof ctx.llm.listProviders === 'function') {
+    try {
+      const list = await ctx.llm.listProviders()
+      if (Array.isArray(list)) {
+        for (const p of list) {
+          const id = typeof p === 'string' ? p : (p?.id || p?.provider)
+          if (id) providers.add(String(id).toLowerCase())
+        }
+      }
+    } catch (_e) {}
+  }
+
+  // 2. ctx.llm.listConfigurableProviders()
+  if (typeof ctx.llm.listConfigurableProviders === 'function') {
+    try {
+      const list = ctx.llm.listConfigurableProviders()
+      if (Array.isArray(list)) {
+        for (const p of list) {
+          const id = typeof p === 'string' ? p : (p?.id || p?.provider)
+          if (id) providers.add(String(id).toLowerCase())
+        }
+      }
+    } catch (_e) {}
+  }
+
+  // 3. Fallback to ctx.llm.providers object or map
+  if (ctx.llm.providers) {
+    if (typeof ctx.llm.providers.keys === 'function') {
+      for (const k of ctx.llm.providers.keys()) providers.add(String(k).toLowerCase())
+    } else if (typeof ctx.llm.providers === 'object') {
+      for (const k of Object.keys(ctx.llm.providers)) providers.add(String(k).toLowerCase())
+    }
+  }
+
+  return providers
+}
+
+/**
+ * Discovers registered models per provider if available.
+ *
+ * @param {object} ctx
+ * @param {Set<string>} [providers]
+ * @returns {Promise<Map<string, Set<string>>>} Map of provider -> Set of model IDs
+ */
+export async function getActiveLlmModels(ctx, providers) {
+  const modelMap = new Map()
+  if (!ctx || !ctx.llm) return modelMap
+
+  const targetProviders = providers || await getActiveLlmProviders(ctx)
+
+  // 1. ctx.llm.listAvailableModels()
+  if (typeof ctx.llm.listAvailableModels === 'function') {
+    try {
+      const all = await ctx.llm.listAvailableModels()
+      if (Array.isArray(all)) {
+        for (const item of all) {
+          const p = String(item?.provider || '').toLowerCase()
+          const m = String(item?.id || item?.model || '').toLowerCase()
+          if (p && m) {
+            if (!modelMap.has(p)) modelMap.set(p, new Set())
+            modelMap.get(p).add(m)
+          }
+        }
+      }
+    } catch (_e) {}
+  }
+
+  // 2. Query each provider with ctx.llm.listModels(p)
+  if (typeof ctx.llm.listModels === 'function') {
+    for (const p of targetProviders) {
+      try {
+        const mList = await Promise.race([
+          ctx.llm.listModels(p),
+          new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 1500)),
+        ])
+        if (Array.isArray(mList)) {
+          if (!modelMap.has(p)) modelMap.set(p, new Set())
+          for (const m of mList) {
+            const mId = typeof m === 'string' ? m : (m?.id || m?.model || m?.name)
+            if (mId) modelMap.get(p).add(String(mId).toLowerCase())
+          }
+        }
+      } catch (_e) {}
+    }
+  }
+
+  return modelMap
+}
+
+/**
+ * Validates a single slot against active providers and models.
+ */
+export function validateSlot(slot, role, activeProviders, modelMap) {
+  if (!slot) return { valid: false, role, reason: 'empty_slot' }
+  const provider = String(slot.provider || '').toLowerCase().trim()
+  const model = String(slot.model || '').toLowerCase().trim()
+  const label = `${slot.provider || 'unknown'}:${slot.model || 'unknown'}`
+
+  if (!provider) {
+    return { valid: false, role, provider: slot.provider, model: slot.model, label, reason: 'missing_provider' }
+  }
+
+  // If we have active providers discovered and provider is not in the list
+  if (activeProviders && activeProviders.size > 0 && !activeProviders.has(provider)) {
+    return { valid: false, role, provider: slot.provider, model: slot.model, label, reason: 'provider_not_found' }
+  }
+
+  // If we have models list for this provider, check if model exists
+  if (modelMap && modelMap.has(provider)) {
+    const models = modelMap.get(provider)
+    if (models && models.size > 0 && !models.has(model)) {
+      return { valid: false, role, provider: slot.provider, model: slot.model, label, reason: 'model_not_found' }
+    }
+  }
+
+  return { valid: true, role, provider: slot.provider, model: slot.model, label }
+}
+
+/**
+ * Validates a preset's reference models, aggregator, and fallbacks.
+ *
+ * @param {object} preset
+ * @param {object} [options]
+ * @returns {object}
+ */
+export function validatePreset(preset, { activeProviders, modelMap } = {}) {
+  if (!preset || typeof preset !== 'object') {
+    return {
+      valid: false,
+      presetName: 'unknown',
+      deadModels: [{ role: 'preset', reason: 'invalid_preset_object' }],
+      warnings: ['Preset is not a valid object'],
+    }
+  }
+
+  const presetName = preset.name || preset.id || 'unnamed'
+  const deadModels = []
+  const warnings = []
+
+  // Check aggregator
+  const aggSlot = preset.aggregator_model || preset.aggregator
+  if (aggSlot) {
+    const res = validateSlot(aggSlot, 'aggregator', activeProviders, modelMap)
+    if (!res.valid) {
+      deadModels.push(res)
+      warnings.push(`Aggregator model ${res.label} has unreachable provider or model (${res.reason})`)
+    }
+  }
+
+  // Check reference models
+  const refs = Array.isArray(preset.references)
+    ? preset.references
+    : (Array.isArray(preset.reference_models) ? preset.reference_models : [])
+
+  if (refs.length === 0) {
+    warnings.push('Preset has no reference models configured')
+  }
+
+  for (let idx = 0; idx < refs.length; idx++) {
+    const slot = refs[idx]
+    const res = validateSlot(slot, `reference_${idx + 1}`, activeProviders, modelMap)
+    if (!res.valid) {
+      deadModels.push(res)
+      warnings.push(`Reference candidate ${res.label} has unreachable provider or model (${res.reason})`)
+    }
+
+    // Check slot fallbacks if any
+    if (Array.isArray(slot?.fallbacks)) {
+      for (const fb of slot.fallbacks) {
+        const fbRes = validateSlot(fb, `fallback_for_${res.label}`, activeProviders, modelMap)
+        if (!fbRes.valid) {
+          deadModels.push(fbRes)
+          warnings.push(`Reserve fallback ${fbRes.label} for ${res.label} is unreachable (${fbRes.reason})`)
+        }
+      }
+    }
+  }
+
+  return {
+    presetId: preset.id || presetName,
+    presetName,
+    valid: deadModels.length === 0,
+    hasDeadModels: deadModels.length > 0,
+    deadModels,
+    warnings,
+    totalSlotsChecked: (aggSlot ? 1 : 0) + refs.length,
+  }
+}
+
+/**
+ * Validates an array of presets and logs warnings if any dead models exist.
+ *
+ * @param {Array<object>} presets
+ * @param {object} ctx
+ * @returns {Promise<object>}
+ */
+export async function validateAllPresets(presets, ctx) {
+  const activeProviders = await getActiveLlmProviders(ctx)
+  const modelMap = await getActiveLlmModels(ctx, activeProviders)
+
+  const presetResults = []
+  let totalDeadModels = 0
+
+  for (const preset of (presets || [])) {
+    const result = validatePreset(preset, { activeProviders, modelMap })
+    presetResults.push(result)
+    if (result.hasDeadModels) {
+      totalDeadModels += result.deadModels.length
+      const deadLabels = result.deadModels.map((m) => `${m.label} (${m.reason})`).join(', ')
+      logger.warn(`[dsh-moa] Preset "${result.presetName}" contains ${result.deadModels.length} dead/unregistered models: ${deadLabels}`)
+    }
+  }
+
+  return {
+    valid: totalDeadModels === 0,
+    activeProviders: Array.from(activeProviders),
+    totalDeadModels,
+    presets: presetResults,
+    validatedAt: Date.now(),
+  }
+}

package/lib/moa-service.js

@@ -152,4 +152,25 @@ export class MoaService {
   emit(event, payload) {
     emitEvent(this.ctx, event, payload)
   }
+
+  /**
+   * Validate presets against active LLM providers in ctx.llm
+   * @param {Array<object>} [presets]
+   * @returns {Promise<object>}
+   */
+  async validatePresets(presets) {
+    const list = presets || this.getPresets()
+    const { validateAllPresets } = await import('./moa-model-validator.js')
+    return validateAllPresets(list, this.ctx)
+  }
+
+  /**
+   * Get list of currently active LLM provider names
+   * @returns {Promise<Array<string>>}
+   */
+  async getActiveProviders() {
+    const { getActiveLlmProviders } = await import('./moa-model-validator.js')
+    const set = await getActiveLlmProviders(this.ctx)
+    return Array.from(set)
+  }
 }