@goodandready/dsh-moa 0.2.68 → 0.2.70

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/client.js CHANGED
@@ -2017,6 +2017,7 @@ window.__ModuleLoader__.load({
2017
2017
  t("aggregator.report_desc") || "Generate downloadable Markdown & JSON reports detailing candidate latency, agreement, test gate, and cost breakdown."
2018
2018
  )
2019
2019
  ),
2020
+ // --- src/client/85-card-aggregator-ext.js ---
2020
2021
  /* Interactive Workbench & File Scoring (#251) */
2021
2022
  React.createElement(
2022
2023
  "div",
@@ -2054,6 +2055,111 @@ window.__ModuleLoader__.load({
2054
2055
  "(" + (t("aggregator.workbench_companion_note") || "safe shadow Git checkpoints before file promotion") + ")"
2055
2056
  )
2056
2057
  ),
2058
+ /* Multimodal Visual MoA Consensus (#250) */
2059
+ React.createElement(
2060
+ "div",
2061
+ { style: { display: "flex", flexDirection: "column", gap: 6, padding: "8px 12px", background: "var(--dsw-alias-bg-layer-2)", borderRadius: 8, border: "1px solid var(--dsw-alias-border-l2)", marginTop: 8 } },
2062
+ React.createElement(
2063
+ "label",
2064
+ { style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer", fontSize: 13, fontWeight: 500 } },
2065
+ React.createElement("input", {
2066
+ type: "checkbox",
2067
+ checked: Boolean(currentPreset.visual_moa_enabled),
2068
+ onChange: (e) => updateCurrentPreset((p) => ({ ...p, visual_moa_enabled: e.target.checked })),
2069
+ style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
2070
+ }),
2071
+ t("aggregator.visual_moa_title") || "\u{1F441}\uFE0F Multimodal Visual MoA Consensus"
2072
+ ),
2073
+ React.createElement(
2074
+ "div",
2075
+ { style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
2076
+ t("aggregator.visual_moa_desc") || "Render UI candidates in Live Canvas and evaluate visual layout fidelity via Vision Bridge VLM."
2077
+ ),
2078
+ React.createElement(
2079
+ "div",
2080
+ { style: { fontSize: 11, color: "var(--dsw-alias-label-tertiary)", marginLeft: 22, marginTop: 2, display: "flex", alignItems: "center", gap: 6, flexWrap: "wrap" } },
2081
+ "\u{1F517} Companion Plugins:",
2082
+ React.createElement(
2083
+ "a",
2084
+ { href: "https://github.com/GooDAnDReaDY/dsh-vision-bridge", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
2085
+ "@goodandready/dsh-vision-bridge"
2086
+ ),
2087
+ "/",
2088
+ React.createElement(
2089
+ "a",
2090
+ { href: "https://github.com/GooDAnDReaDY/dsh-live-canvas", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
2091
+ "@goodandready/dsh-live-canvas"
2092
+ )
2093
+ )
2094
+ ),
2095
+ /* Quota-Aware Provider Balancing (#253) */
2096
+ React.createElement(
2097
+ "div",
2098
+ { style: { display: "flex", flexDirection: "column", gap: 6, padding: "8px 12px", background: "var(--dsw-alias-bg-layer-2)", borderRadius: 8, border: "1px solid var(--dsw-alias-border-l2)", marginTop: 8 } },
2099
+ React.createElement(
2100
+ "label",
2101
+ { style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer", fontSize: 13, fontWeight: 500 } },
2102
+ React.createElement("input", {
2103
+ type: "checkbox",
2104
+ checked: Boolean(currentPreset.quota_aware_balancing_enabled),
2105
+ onChange: (e) => updateCurrentPreset((p) => ({ ...p, quota_aware_balancing_enabled: e.target.checked })),
2106
+ style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
2107
+ }),
2108
+ t("aggregator.quota_balancing_title") || "\u2696\uFE0F Quota-Aware Provider Balancing"
2109
+ ),
2110
+ React.createElement(
2111
+ "div",
2112
+ { style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
2113
+ t("aggregator.quota_balancing_desc") || "Automatically substitute low-quota candidate models in Layer 1 to preserve rate limits for the final aggregator."
2114
+ ),
2115
+ React.createElement(
2116
+ "div",
2117
+ { style: { fontSize: 11, color: "var(--dsw-alias-label-tertiary)", marginLeft: 22, marginTop: 2, display: "flex", alignItems: "center", gap: 6, flexWrap: "wrap" } },
2118
+ "\u{1F517} Companion Plugins:",
2119
+ React.createElement(
2120
+ "a",
2121
+ { href: "https://github.com/GooDAnDReaDY/dsh-key-limits", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
2122
+ "@goodandready/dsh-key-limits"
2123
+ ),
2124
+ "/",
2125
+ React.createElement(
2126
+ "a",
2127
+ { href: "https://github.com/GooDAnDReaDY/dsh-subscriptions", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
2128
+ "@goodandready/dsh-subscriptions"
2129
+ )
2130
+ )
2131
+ ),
2132
+ /* Self-Healing Post-Promotion Build Loop (#252) */
2133
+ React.createElement(
2134
+ "div",
2135
+ { style: { display: "flex", flexDirection: "column", gap: 6, padding: "8px 12px", background: "var(--dsw-alias-bg-layer-2)", borderRadius: 8, border: "1px solid var(--dsw-alias-border-l2)", marginTop: 8 } },
2136
+ React.createElement(
2137
+ "label",
2138
+ { style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer", fontSize: 13, fontWeight: 500 } },
2139
+ React.createElement("input", {
2140
+ type: "checkbox",
2141
+ checked: Boolean(currentPreset.self_healing_enabled),
2142
+ onChange: (e) => updateCurrentPreset((p) => ({ ...p, self_healing_enabled: e.target.checked })),
2143
+ style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
2144
+ }),
2145
+ t("aggregator.self_healing_title") || "\u{1FA7A} Self-Healing Post-Promotion Build Loop"
2146
+ ),
2147
+ React.createElement(
2148
+ "div",
2149
+ { style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
2150
+ t("aggregator.self_healing_desc") || "Verify promoted files with a test command, request hotfix patches from the aggregator on failure, or auto-rollback to the Time Machine checkpoint."
2151
+ ),
2152
+ React.createElement(
2153
+ "div",
2154
+ { style: { fontSize: 11, color: "var(--dsw-alias-label-tertiary)", marginLeft: 22, marginTop: 2, display: "flex", alignItems: "center", gap: 6 } },
2155
+ "\u{1F517} Companion Plugin:",
2156
+ React.createElement(
2157
+ "a",
2158
+ { href: "https://github.com/GooDAnDReaDY/dsh-time-machine", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
2159
+ "@goodandready/dsh-time-machine"
2160
+ )
2161
+ )
2162
+ ),
2057
2163
  /* Graceful Degradation & Local Fallback (#175, #176) */
2058
2164
  React.createElement(
2059
2165
  "div",
@@ -2511,6 +2617,7 @@ window.__ModuleLoader__.load({
2511
2617
  React.createElement("th", { style: { padding: "6px 8px" } }, t("leaderboard.runs")),
2512
2618
  React.createElement("th", { style: { padding: "6px 8px" } }, t("leaderboard.wins")),
2513
2619
  React.createElement("th", { style: { padding: "6px 8px" } }, t("leaderboard.winrate")),
2620
+ React.createElement("th", { style: { padding: "6px 8px" } }, "Elo"),
2514
2621
  React.createElement("th", { style: { padding: "6px 10px" } }, t("leaderboard.cost"))
2515
2622
  )
2516
2623
  ),
@@ -2525,6 +2632,7 @@ window.__ModuleLoader__.load({
2525
2632
  React.createElement("td", { style: { padding: "6px 8px", color: "var(--dsw-alias-label-secondary)" } }, String(m.runs)),
2526
2633
  React.createElement("td", { style: { padding: "6px 8px", color: "var(--dsw-alias-state-success-primary)" } }, String(m.wins)),
2527
2634
  React.createElement("td", { style: { padding: "6px 8px", fontWeight: 600 } }, (m.winRate ?? 0) + "%"),
2635
+ React.createElement("td", { style: { padding: "6px 8px", fontWeight: 600, color: "var(--dsw-alias-state-brand-primary)" } }, String(m.elo ?? 1500)),
2528
2636
  React.createElement("td", { style: { padding: "6px 10px", color: "var(--dsw-alias-label-secondary)" } }, m.avgCostUsd > 0 ? "$" + m.avgCostUsd.toFixed(4) : "Free")
2529
2637
  )
2530
2638
  )
package/lib/history.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { classifyEloCategory, computeEloFromRuns, INITIAL_ELO } from './moa-elo.js'
1
2
  import { logger } from './logger.js'
2
3
  /**
3
4
  * MoA Run History & Analytics Module
@@ -355,6 +356,7 @@ export function getMoaLeaderboard(filePath = getDefaultHistoryFile(), options =
355
356
  _historyCache.totalCount = totalRuns
356
357
 
357
358
  const modelStats = new Map()
359
+ const parsedRuns = []
358
360
  let filteredRuns = 0
359
361
  for (const line of lines) {
360
362
  try {
@@ -362,6 +364,7 @@ export function getMoaLeaderboard(filePath = getDefaultHistoryFile(), options =
362
364
  if (!item) continue
363
365
  if (presetFilter && presetFilter !== 'all' && item.preset !== presetFilter) continue
364
366
  filteredRuns++
367
+ parsedRuns.push(item)
365
368
 
366
369
  const rawWinner = item.winnerModel ? String(item.winnerModel).trim() : null
367
370
  const cleanWinner = rawWinner ? rawWinner.replace(/\s*\[fallback for .*\]$/i, '').trim() : null
@@ -415,13 +418,25 @@ export function getMoaLeaderboard(filePath = getDefaultHistoryFile(), options =
415
418
  }
416
419
  }
417
420
 
421
+ const { globalRatings, categoryRatings } = computeEloFromRuns(parsedRuns)
418
422
  const models = Array.from(modelStats.values())
419
- .map((st) => ({
420
- ...st,
421
- winRate: st.runs > 0 ? Number(((st.wins / st.runs) * 100).toFixed(1)) : 0,
422
- avgTokens: st.runs > 0 ? Math.round(st.totalTokens / st.runs) : 0,
423
- avgCostUsd: st.runs > 0 ? Number((st.totalCostUsd / st.runs).toFixed(4)) : 0,
424
- }))
423
+ .map((st) => {
424
+ const elo = Math.round(globalRatings.get(st.modelKey) ?? INITIAL_ELO)
425
+ const categoryElo = {}
426
+ for (const [cat, cMap] of categoryRatings.entries()) {
427
+ if (cMap.has(st.modelKey)) {
428
+ categoryElo[cat] = Math.round(cMap.get(st.modelKey))
429
+ }
430
+ }
431
+ return {
432
+ ...st,
433
+ elo,
434
+ categoryElo,
435
+ winRate: st.runs > 0 ? Number(((st.wins / st.runs) * 100).toFixed(1)) : 0,
436
+ avgTokens: st.runs > 0 ? Math.round(st.totalTokens / st.runs) : 0,
437
+ avgCostUsd: st.runs > 0 ? Number((st.totalCostUsd / st.runs).toFixed(4)) : 0,
438
+ }
439
+ })
425
440
  .sort((a, b) => b.wins - a.wins || b.winRate - a.winRate)
426
441
 
427
442
  const result = {
package/lib/moa-elo.js ADDED
@@ -0,0 +1,189 @@
1
+ /**
2
+ * Empirical Elo Rating and Active Learning Leaderboard by Task Category (#249)
3
+ * Calculates pairwise Elo ratings (K-factor = 32, baseline = 1500) segmented by
4
+ * task category (frontend, backend, refactoring, testing, sql, general),
5
+ * and exposes category leaders to Smart Routing.
6
+ */
7
+
8
+ export const INITIAL_ELO = 1500
9
+ export const DEFAULT_K_FACTOR = 32
10
+
11
+ const CATEGORY_PATTERNS = [
12
+ { category: 'sql', regex: /\b(sql|postgres|sqlite|mysql|query|database|schema|migration|prisma|index|table)\b/i },
13
+ { category: 'frontend', regex: /\b(frontend|ui|css|html|react|vue|tailwind|component|button|layout|modal|svg|dom)\b/i },
14
+ { category: 'testing', regex: /\b(test|unit\s*test|e2e|vitest|jest|playwright|assert|coverage|mock|tdd)\b/i },
15
+ { category: 'refactoring', regex: /\b(refactor|clean|simplify|dedup|modular|rename|extract|decompose|yagni)\b/i },
16
+ { category: 'backend', regex: /\b(backend|api|endpoint|server|route|auth|jwt|middleware|express|http|rest|webhook|stream)\b/i },
17
+ ]
18
+
19
+ /**
20
+ * Classifies a prompt into one of the standard Elo leaderboard categories.
21
+ *
22
+ * @param {string} prompt
23
+ * @returns {'frontend' | 'backend' | 'refactoring' | 'testing' | 'sql' | 'general'}
24
+ */
25
+ export function classifyEloCategory(prompt = '') {
26
+ const text = String(prompt || '').trim()
27
+ if (!text) return 'general'
28
+ for (const rule of CATEGORY_PATTERNS) {
29
+ if (rule.regex.test(text)) return rule.category
30
+ }
31
+ return 'general'
32
+ }
33
+
34
+ /**
35
+ * Calculates expected win probability of player A against player B.
36
+ */
37
+ export function expectedEloScore(ratingA, ratingB) {
38
+ return 1 / (1 + Math.pow(10, (ratingB - ratingA) / 400))
39
+ }
40
+
41
+ /**
42
+ * Updates pairwise Elo ratings when winnerKey beats all other participantKeys in a round.
43
+ *
44
+ * @param {Map<string, number>} ratingMap
45
+ * @param {string} winnerKey
46
+ * @param {string[]} participantKeys
47
+ * @param {number} [kFactor=32]
48
+ */
49
+ export function applyRoundEloUpdate(ratingMap, winnerKey, participantKeys = [], kFactor = DEFAULT_K_FACTOR) {
50
+ if (!winnerKey || participantKeys.length < 2) return
51
+ const opponents = participantKeys.filter((k) => k && k !== winnerKey)
52
+ if (opponents.length === 0) return
53
+
54
+ // Normalize K-factor across the number of pairwise comparisons in the multi-candidate round
55
+ const effectiveK = kFactor / opponents.length
56
+ let winnerRating = ratingMap.get(winnerKey) ?? INITIAL_ELO
57
+ let winnerDelta = 0
58
+
59
+ for (const oppKey of opponents) {
60
+ const oppRating = ratingMap.get(oppKey) ?? INITIAL_ELO
61
+ const expWin = expectedEloScore(winnerRating, oppRating)
62
+ const expOpp = 1 - expWin
63
+
64
+ winnerDelta += effectiveK * (1 - expWin)
65
+ const nextOpp = oppRating + effectiveK * (0 - expOpp)
66
+ ratingMap.set(oppKey, nextOpp)
67
+ }
68
+
69
+ ratingMap.set(winnerKey, winnerRating + winnerDelta)
70
+ }
71
+
72
+ /**
73
+ * Computes global and per-category Elo ratings across a chronological list of MoA runs.
74
+ *
75
+ * @param {Array<object>} runs - Array of history run objects (oldest to newest)
76
+ * @param {number} [kFactor=32]
77
+ * @returns {{
78
+ * globalRatings: Map<string, number>,
79
+ * categoryRatings: Map<string, Map<string, number>>
80
+ * }}
81
+ */
82
+ export function computeEloFromRuns(runs = [], kFactor = DEFAULT_K_FACTOR) {
83
+ const globalRatings = new Map()
84
+ const categoryRatings = new Map()
85
+
86
+ for (const run of runs) {
87
+ if (!run || !Array.isArray(run.candidates) || run.candidates.length === 0) continue
88
+
89
+ const category = run.category || classifyEloCategory(run.prompt || '')
90
+ if (!categoryRatings.has(category)) {
91
+ categoryRatings.set(category, new Map())
92
+ }
93
+ const catMap = categoryRatings.get(category)
94
+
95
+ const participantKeys = []
96
+ let winnerKey = null
97
+
98
+ const rawWinner = run.winnerModel ? String(run.winnerModel).trim() : null
99
+ const cleanWinner = rawWinner ? rawWinner.replace(/\s*\[fallback for .*\]$/i, '').trim() : null
100
+ const normWinner = cleanWinner && cleanWinner !== 'none' && cleanWinner !== 'unknown'
101
+ ? cleanWinner.replace(/[\/:]/g, ':').toLowerCase()
102
+ : null
103
+
104
+ for (let idx = 0; idx < run.candidates.length; idx++) {
105
+ const cand = run.candidates[idx]
106
+ const key = cand.provider && cand.model ? `${cand.provider}:${cand.model}` : (cand.model || 'unknown')
107
+ participantKeys.push(key)
108
+ if (!globalRatings.has(key)) globalRatings.set(key, INITIAL_ELO)
109
+ if (!catMap.has(key)) catMap.set(key, INITIAL_ELO)
110
+
111
+ if (!winnerKey) {
112
+ if (normWinner) {
113
+ const normKey = key.replace(/[\/:]/g, ':').toLowerCase()
114
+ const normModel = String(cand.model || '').replace(/[\/:]/g, ':').toLowerCase()
115
+ if (normWinner === normKey || normWinner === normModel || normWinner.endsWith(`:${normModel}`)) {
116
+ winnerKey = key
117
+ }
118
+ }
119
+ if (!winnerKey && typeof run.winnerIndex === 'number' && run.winnerIndex > 0) {
120
+ if (cand.index === run.winnerIndex || idx === run.winnerIndex - 1) {
121
+ winnerKey = key
122
+ }
123
+ }
124
+ }
125
+ }
126
+
127
+ if (winnerKey && participantKeys.length >= 2) {
128
+ applyRoundEloUpdate(globalRatings, winnerKey, participantKeys, kFactor)
129
+ applyRoundEloUpdate(catMap, winnerKey, participantKeys, kFactor)
130
+ }
131
+ }
132
+
133
+ return { globalRatings, categoryRatings }
134
+ }
135
+
136
+ /**
137
+ * Selects the best preset for a prompt category based on historical Elo ratings of its reference models.
138
+ *
139
+ * @param {string} prompt
140
+ * @param {Array<object>} presets
141
+ * @param {object} leaderboard - Result from getMoaLeaderboard()
142
+ * @returns {{ presetName: string, category: string, avgElo: number } | null}
143
+ */
144
+ export function selectBestPresetByElo(prompt = '', presets = [], leaderboard = null) {
145
+ if (!leaderboard || !Array.isArray(leaderboard.models) || leaderboard.models.length === 0) return null
146
+ if (!Array.isArray(presets) || presets.length === 0) return null
147
+
148
+ const category = classifyEloCategory(prompt)
149
+ const modelMap = new Map()
150
+ for (const m of leaderboard.models) {
151
+ modelMap.set(m.modelKey, m)
152
+ }
153
+
154
+ let bestPreset = null
155
+ let bestElo = -Infinity
156
+
157
+ for (const preset of presets) {
158
+ if (preset.enabled === false) continue
159
+ const refs = Array.isArray(preset.reference_models) ? preset.reference_models : []
160
+ if (refs.length === 0) continue
161
+
162
+ let sumElo = 0
163
+ let matchedCount = 0
164
+ for (const ref of refs) {
165
+ const key = ref.provider && ref.model ? `${ref.provider}:${ref.model}` : ref.model
166
+ const st = modelMap.get(key)
167
+ if (st) {
168
+ const catElo = st.categoryElo?.[category] ?? st.elo ?? INITIAL_ELO
169
+ sumElo += catElo
170
+ matchedCount++
171
+ }
172
+ }
173
+
174
+ if (matchedCount > 0) {
175
+ const avgElo = sumElo / matchedCount
176
+ if (avgElo > bestElo && avgElo > INITIAL_ELO) {
177
+ bestElo = avgElo
178
+ bestPreset = preset.name
179
+ }
180
+ }
181
+ }
182
+
183
+ if (!bestPreset) return null
184
+ return {
185
+ presetName: bestPreset,
186
+ category,
187
+ avgElo: Math.round(bestElo),
188
+ }
189
+ }
@@ -0,0 +1,188 @@
1
+ /**
2
+ * Quota-Aware Provider Balancing (#253)
3
+ * Queries provider/model rate limits and quota health via dsh-key-limits and dsh-subscriptions.
4
+ * Dynamically substitutes exhausted or low-quota reference slots with healthy fallback models
5
+ * to protect the final aggregator quota and prevent 429 pipeline aborts.
6
+ */
7
+
8
+ import { service } from './services.js'
9
+ import { logger } from './logger.js'
10
+ import { slotLabel } from './moa-prompts.js'
11
+
12
+ const DEFAULT_FALLBACK_POOL = [
13
+ { provider: 'deepseek', model: 'deepseek-reasoner' },
14
+ { provider: 'deepseek', model: 'deepseek-chat' },
15
+ { provider: 'qwen', model: 'qwen-2.5-coder-32b-instruct' },
16
+ { provider: 'openrouter', model: 'deepseek/deepseek-r1' },
17
+ { provider: 'ollama', model: 'qwen2.5-coder:32b' },
18
+ ]
19
+
20
+ /**
21
+ * Inspects quota status for a given { provider, model } slot via dsh-key-limits or dsh-subscriptions.
22
+ *
23
+ * @param {object} ctx - Cordis context
24
+ * @param {{ provider: string, model: string }} slot
25
+ * @param {object} [options]
26
+ * @returns {Promise<{ exhausted: boolean, remainingRatio: number, reason: string | null }>}
27
+ */
28
+ export async function checkSlotQuotaStatus(ctx, slot, options = {}) {
29
+ const provider = String(slot?.provider || '').trim()
30
+ const model = String(slot?.model || '').trim()
31
+ if (!provider) return { exhausted: false, remainingRatio: 1, reason: null }
32
+
33
+ const minRemainingRatio = Number.isFinite(options.minRemainingRatio) ? options.minRemainingRatio : 0.1
34
+
35
+ const keyLimits = options.keyLimitsService || service(ctx, 'keyLimits')
36
+ if (keyLimits) {
37
+ try {
38
+ let status = null
39
+ if (typeof keyLimits.getQuotaStatus === 'function') {
40
+ status = await keyLimits.getQuotaStatus(provider, model)
41
+ } else if (typeof keyLimits.checkLimit === 'function') {
42
+ status = await keyLimits.checkLimit(provider, model)
43
+ } else if (typeof keyLimits.getProviderStatus === 'function') {
44
+ status = await keyLimits.getProviderStatus(provider)
45
+ }
46
+
47
+ if (status) {
48
+ if (status.exhausted === true || status.rateLimited === true || status.blocked === true) {
49
+ return {
50
+ exhausted: true,
51
+ remainingRatio: 0,
52
+ reason: status.reason || 'Quota exhausted or rate-limited in dsh-key-limits',
53
+ }
54
+ }
55
+ const rem = Number(status.remaining ?? status.remainingRequests ?? status.remainingTokens)
56
+ const lim = Number(status.limit ?? status.maxRequests ?? status.maxTokens)
57
+ if (Number.isFinite(rem) && Number.isFinite(lim) && lim > 0) {
58
+ const ratio = rem / lim
59
+ if (ratio < minRemainingRatio) {
60
+ return {
61
+ exhausted: true,
62
+ remainingRatio: ratio,
63
+ reason: `Low quota remaining (${Math.round(ratio * 100)}%) in dsh-key-limits`,
64
+ }
65
+ }
66
+ return { exhausted: false, remainingRatio: ratio, reason: null }
67
+ }
68
+ if (Number.isFinite(status.remainingRatio)) {
69
+ if (status.remainingRatio < minRemainingRatio) {
70
+ return {
71
+ exhausted: true,
72
+ remainingRatio: status.remainingRatio,
73
+ reason: `Low quota ratio (${Math.round(status.remainingRatio * 100)}%) in dsh-key-limits`,
74
+ }
75
+ }
76
+ return { exhausted: false, remainingRatio: status.remainingRatio, reason: null }
77
+ }
78
+ }
79
+ } catch (err) {
80
+ logger.warn(`[dsh-moa:quota] Error querying dsh-key-limits for ${provider}:${model}:`, err?.message || err)
81
+ }
82
+ }
83
+
84
+ const subscriptions = options.subscriptionsService || service(ctx, 'subscriptions')
85
+ if (subscriptions) {
86
+ try {
87
+ let subStatus = null
88
+ if (typeof subscriptions.getSubscriptionStatus === 'function') {
89
+ subStatus = await subscriptions.getSubscriptionStatus(provider, model)
90
+ } else if (typeof subscriptions.checkQuota === 'function') {
91
+ subStatus = await subscriptions.checkQuota(provider, model)
92
+ }
93
+
94
+ if (subStatus) {
95
+ if (subStatus.exhausted === true || subStatus.active === false || subStatus.expired === true) {
96
+ return {
97
+ exhausted: true,
98
+ remainingRatio: 0,
99
+ reason: subStatus.reason || 'Subscription quota exhausted or inactive in dsh-subscriptions',
100
+ }
101
+ }
102
+ if (Number.isFinite(subStatus.remainingRatio) && subStatus.remainingRatio < minRemainingRatio) {
103
+ return {
104
+ exhausted: true,
105
+ remainingRatio: subStatus.remainingRatio,
106
+ reason: `Low subscription quota (${Math.round(subStatus.remainingRatio * 100)}%) in dsh-subscriptions`,
107
+ }
108
+ }
109
+ }
110
+ } catch (err) {
111
+ logger.warn(`[dsh-moa:quota] Error querying dsh-subscriptions for ${provider}:${model}:`, err?.message || err)
112
+ }
113
+ }
114
+
115
+ return { exhausted: false, remainingRatio: 1, reason: null }
116
+ }
117
+
118
+ /**
119
+ * Balances candidate reference slots before Layer 1 execution.
120
+ * Preserves quota for the aggregator by substituting low-quota/exhausted slots
121
+ * with healthy fallback models.
122
+ *
123
+ * @param {Array<{ provider: string, model: string }>} references
124
+ * @param {object} options
125
+ * @returns {Promise<{ balancedReferences: Array<{ provider: string, model: string }>, substitutions: Array<object> }>}
126
+ */
127
+ export async function balanceCandidateQuotas(references = [], options = {}) {
128
+ if (!Array.isArray(references) || references.length === 0) {
129
+ return { balancedReferences: [], substitutions: [] }
130
+ }
131
+
132
+ const ctx = options.ctx || null
133
+ const aggregator = options.aggregator || null
134
+ const customFallbacks = Array.isArray(options.fallbackPool) && options.fallbackPool.length > 0
135
+ ? options.fallbackPool
136
+ : DEFAULT_FALLBACK_POOL
137
+
138
+ const usedKeys = new Set(references.map((r) => slotLabel(r)))
139
+ const balancedReferences = []
140
+ const substitutions = []
141
+
142
+ for (let i = 0; i < references.length; i++) {
143
+ const slot = references[i]
144
+ const label = slotLabel(slot)
145
+
146
+ // Reserve higher threshold (25%) if this candidate uses the exact same provider:model as the final aggregator
147
+ const isSameAsAggregator = aggregator && slotLabel(aggregator) === label
148
+ const minRemainingRatio = isSameAsAggregator ? 0.25 : (options.minRemainingRatio ?? 0.1)
149
+
150
+ const status = await checkSlotQuotaStatus(ctx, slot, {
151
+ ...options,
152
+ minRemainingRatio,
153
+ })
154
+
155
+ if (!status.exhausted) {
156
+ balancedReferences.push(slot)
157
+ continue
158
+ }
159
+
160
+ // Find a healthy replacement from customFallbacks
161
+ let replacement = null
162
+ for (const candidateFallback of customFallbacks) {
163
+ const fbLabel = slotLabel(candidateFallback)
164
+ if (fbLabel === label) continue
165
+ const fbStatus = await checkSlotQuotaStatus(ctx, candidateFallback, options)
166
+ if (!fbStatus.exhausted) {
167
+ replacement = { ...slot, provider: candidateFallback.provider, model: candidateFallback.model, replacedFrom: label }
168
+ usedKeys.add(fbLabel)
169
+ break
170
+ }
171
+ }
172
+
173
+ if (replacement) {
174
+ balancedReferences.push(replacement)
175
+ substitutions.push({
176
+ index: i + 1,
177
+ original: slot,
178
+ replacement: { provider: replacement.provider, model: replacement.model },
179
+ reason: status.reason,
180
+ })
181
+ logger.info(`[dsh-moa:quota] Replaced slot #${i + 1} (${label}) -> ${slotLabel(replacement)} (${status.reason})`)
182
+ } else {
183
+ balancedReferences.push(slot)
184
+ }
185
+ }
186
+
187
+ return { balancedReferences, substitutions }
188
+ }
package/lib/moa-router.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { selectBestPresetByElo } from './moa-elo.js'
1
2
  /**
2
3
  * Smart Preset Router for Mixture of Agents (MoA).
3
4
  * Supports heuristic keyword classification and optional LLM zero-shot routing.
@@ -32,15 +33,18 @@ export function classifyPromptIntent(prompt = '') {
32
33
  /**
33
34
  * Resolves the optimal preset for a given user prompt.
34
35
  */
35
- export async function resolvePresetForPrompt({
36
- prompt = '',
37
- presets = [],
38
- defaultPreset = 'default',
39
- routingModel = null,
40
- callLlm = null,
41
- enabled = false,
42
- timeoutMs = 2000,
43
- }) {
36
+ export async function resolvePresetForPrompt(arg1 = {}, arg2, arg3, arg4 = {}) {
37
+ const isPositional = typeof arg1 === "string"
38
+ const prompt = isPositional ? arg1 : (arg1.prompt || "")
39
+ const presets = isPositional ? (arg2 || []) : (arg1.presets || [])
40
+ const defaultPreset = isPositional ? (arg3 || "default") : (arg1.defaultPreset || "default")
41
+ const opts = isPositional ? (arg4 || {}) : arg1
42
+ const routingModel = opts.routingModel || null
43
+ const callLlm = opts.callLlm || null
44
+ const enabled = Boolean(opts.enabled ?? opts.smartRoutingEnabled ?? false)
45
+ const timeoutMs = opts.timeoutMs ?? 2000
46
+ const leaderboard = opts.leaderboard || null
47
+ const eloRankingEnabled = opts.eloRankingEnabled !== false
44
48
  const availableNames = (presets || []).map((p) => p.name).filter(Boolean)
45
49
  const safeDefault = availableNames.includes(defaultPreset) ? defaultPreset : (availableNames[0] || 'default')
46
50
 
@@ -87,6 +91,20 @@ Respond with ONLY the chosen preset name and nothing else.`
87
91
  }
88
92
  }
89
93
 
94
+ // 1.5. Active Learning Elo Leaderboard routing (#249)
95
+ if (leaderboard && eloRankingEnabled) {
96
+ const eloChoice = selectBestPresetByElo(prompt, presets, leaderboard)
97
+ if (eloChoice && availableNames.includes(eloChoice.presetName)) {
98
+ return {
99
+ presetName: eloChoice.presetName,
100
+ reason: "elo_active_learning",
101
+ category: eloChoice.category,
102
+ avgElo: eloChoice.avgElo,
103
+ isAutoRouted: true,
104
+ }
105
+ }
106
+ }
107
+
90
108
  // 2. Keyword heuristic classifier
91
109
  const heuristic = classifyPromptIntent(prompt)
92
110
  if (heuristic && availableNames.includes(heuristic)) {
package/lib/moa-schema.js CHANGED
@@ -66,6 +66,12 @@ export const PresetSchema = z.object({
66
66
  uncertainty_scoring_enabled: z.boolean().default(false),
67
67
  subagent_swarm_enabled: z.boolean().default(false),
68
68
  adaptive_personas_enabled: z.boolean().default(false),
69
+ quota_aware_balancing_enabled: z.boolean().default(false),
70
+ self_healing_enabled: z.boolean().default(false),
71
+ self_healing_max_retries: z.number().default(2),
72
+ self_healing_command: z.string().default("npm test"),
73
+ visual_moa_enabled: z.boolean().default(false),
74
+ elo_ranking_enabled: z.boolean().default(true),
69
75
  })
70
76
 
71
77
  export const Config = z.object({
@@ -78,6 +84,12 @@ export const Config = z.object({
78
84
  uncertainty_scoring_enabled: z.boolean().default(false).volatile(),
79
85
  subagent_swarm_enabled: z.boolean().default(false).volatile(),
80
86
  adaptive_personas_enabled: z.boolean().default(false).volatile(),
87
+ quota_aware_balancing_enabled: z.boolean().default(false).volatile(),
88
+ self_healing_enabled: z.boolean().default(false).volatile(),
89
+ self_healing_max_retries: z.number().default(2).volatile(),
90
+ self_healing_command: z.string().default("npm test").volatile(),
91
+ visual_moa_enabled: z.boolean().default(false).volatile(),
92
+ elo_ranking_enabled: z.boolean().default(true).volatile(),
81
93
  smart_routing_model: ModelSlotSchema.default({ provider: '', model: '' }).volatile(),
82
94
  prices: z.dict(PriceRow).default({}).volatile(),
83
95
  presets: z.array(PresetSchema).default([
@@ -0,0 +1,258 @@
1
+ /**
2
+ * Self-Healing Post-Promotion Build Loop with dsh-time-machine Auto-Rollback (#252)
3
+ * Executes verification command after file promotion, dispatches compiler/test error logs
4
+ * to the MoA aggregator for hotfix generation up to maxRetries times, and automatically
5
+ * rolls back to the pre-promotion dsh-time-machine snapshot if verification still fails.
6
+ */
7
+
8
+ import { execFile } from 'node:child_process'
9
+ import { promisify } from 'node:util'
10
+ import fs from 'node:fs/promises'
11
+ import path from 'node:path'
12
+ import { logger } from './logger.js'
13
+ import { bestEffort } from './best-effort.js'
14
+ import { extractFileBlocks } from './workspace/fs.js'
15
+ import { isSafeRelativePath, assertPathContained, isLoopbackUrl } from './workspace/security.js'
16
+
17
+ const execFileAsync = promisify(execFile)
18
+
19
+ /**
20
+ * Runs a verification command in the target workspace.
21
+ *
22
+ * @param {string} cwd
23
+ * @param {string} commandStr
24
+ * @param {object} [options]
25
+ * @returns {Promise<{ passed: boolean, exitCode: number, output: string }>}
26
+ */
27
+ export async function runVerificationCommand(cwd, commandStr = 'npm test', options = {}) {
28
+ if (typeof options.execFn === 'function') {
29
+ return await options.execFn(cwd, commandStr)
30
+ }
31
+
32
+ const trimmed = String(commandStr || 'npm test').trim()
33
+ const isWin = process.platform === 'win32'
34
+ const shell = isWin ? 'cmd.exe' : '/bin/sh'
35
+ const shellArgs = isWin ? ['/d', '/s', '/c', trimmed] : ['-c', trimmed]
36
+
37
+ try {
38
+ const { stdout, stderr } = await execFileAsync(shell, shellArgs, {
39
+ cwd,
40
+ timeout: (options.timeoutSec ?? 60) * 1000,
41
+ maxBuffer: 4 * 1024 * 1024,
42
+ signal: options.signal,
43
+ })
44
+ return {
45
+ passed: true,
46
+ exitCode: 0,
47
+ output: `${stdout || ''}\n${stderr || ''}`.trim(),
48
+ }
49
+ } catch (err) {
50
+ const out = `${err?.stdout || ''}\n${err?.stderr || ''}\n${err?.message || ''}`.trim()
51
+ return {
52
+ passed: false,
53
+ exitCode: typeof err?.code === 'number' ? err.code : 1,
54
+ output: out.slice(0, 12000),
55
+ }
56
+ }
57
+ }
58
+
59
+ /**
60
+ * Triggers an automatic rollback to a pre-promotion dsh-time-machine snapshot.
61
+ *
62
+ * @param {string} baseDir
63
+ * @param {object} snapshot
64
+ * @param {object} [options]
65
+ * @returns {Promise<{ rolledBack: boolean, snapshotId: string | null }>}
66
+ */
67
+ export async function rollbackToTimeMachineSnapshot(baseDir, snapshot, options = {}) {
68
+ const snapshotId = typeof snapshot === 'string'
69
+ ? snapshot
70
+ : (snapshot?.id || snapshot?.snapshotId || snapshot?.commit || snapshot?.hash || null)
71
+
72
+ if (typeof options.rollbackFn === 'function') {
73
+ const res = await bestEffort('custom rollbackFn', () => options.rollbackFn(baseDir, snapshotId, snapshot))
74
+ return { rolledBack: Boolean(res), snapshotId }
75
+ }
76
+
77
+ if (options.timeMachineEngine) {
78
+ const fn = options.timeMachineEngine.restoreSnapshot || options.timeMachineEngine.rollback
79
+ if (typeof fn === 'function') {
80
+ const res = await bestEffort('dsh-time-machine direct restoreSnapshot', () =>
81
+ fn.call(options.timeMachineEngine, snapshotId, { cwd: baseDir })
82
+ )
83
+ if (res) return { rolledBack: true, snapshotId }
84
+ }
85
+ }
86
+
87
+ const port = options.port || options.webServerPort || process.env.DSH_PORT || process.env.PORT
88
+ if (!port && !options.timeMachineRollbackUrl) {
89
+ logger.warn('[dsh-moa:self-healing] Cannot rollback: webServerPort is unknown and no timeMachineEngine/rollbackFn provided')
90
+ return { rolledBack: false, snapshotId }
91
+ }
92
+
93
+ const endpoint = options.timeMachineRollbackUrl || `http://127.0.0.1:${port}/dsh-time-machine/restore`
94
+ if (!isLoopbackUrl(endpoint)) {
95
+ logger.warn(`[dsh-moa:self-healing] Blocked non-loopback rollback URL: ${endpoint}`)
96
+ return { rolledBack: false, snapshotId }
97
+ }
98
+
99
+ const restored = await bestEffort('dsh-time-machine HTTP rollback', async () => {
100
+ const res = await fetch(endpoint, {
101
+ method: 'POST',
102
+ headers: { 'Content-Type': 'application/json' },
103
+ body: JSON.stringify({ id: snapshotId, snapshotId, cwd: baseDir }),
104
+ signal: AbortSignal.timeout(options.timeoutMs ?? 5000),
105
+ redirect: 'error',
106
+ })
107
+ return res.ok
108
+ })
109
+
110
+ return { rolledBack: Boolean(restored), snapshotId }
111
+ }
112
+
113
+ /**
114
+ * Applies extracted hotfix files directly to the workspace directory.
115
+ */
116
+ async function applyHotfixFiles(cwd, files = []) {
117
+ const written = []
118
+ for (const f of files) {
119
+ const rel = f.relativePath || f.path
120
+ if (!rel || !isSafeRelativePath(rel)) continue
121
+ const targetPath = assertPathContained(cwd, rel)
122
+ await fs.mkdir(path.dirname(targetPath), { recursive: true })
123
+ await fs.writeFile(targetPath, f.content, 'utf8')
124
+ written.push(rel)
125
+ }
126
+ return written
127
+ }
128
+
129
+ /**
130
+ * Runs the post-promotion self-healing build loop.
131
+ *
132
+ * @param {object} params
133
+ * @returns {Promise<{
134
+ * enabled: boolean,
135
+ * passed: boolean,
136
+ * attempts: number,
137
+ * rolledBack: boolean,
138
+ * snapshotId: string | null,
139
+ * hotfixFiles: string[],
140
+ * lastOutput: string
141
+ * }>}
142
+ */
143
+ export async function runSelfHealingLoop({
144
+ cwd,
145
+ command = 'npm test',
146
+ maxRetries = 2,
147
+ aggregatorSlot,
148
+ callLlm,
149
+ userPrompt = '',
150
+ promotedFiles = [],
151
+ checkpoint = null,
152
+ promoteOpts = {},
153
+ onProgress = null,
154
+ signal = null,
155
+ execFn = null,
156
+ rollbackFn = null,
157
+ }) {
158
+ const retries = Math.max(0, Math.min(5, Number(maxRetries ?? 2)))
159
+ const hotfixFiles = new Set()
160
+
161
+ if (typeof onProgress === 'function') {
162
+ onProgress(`🩺 *Self-Healing Verification: Running \`${command}\`...*\n`)
163
+ }
164
+
165
+ let check = await runVerificationCommand(cwd, command, { execFn, signal })
166
+ if (check.passed) {
167
+ return {
168
+ enabled: true,
169
+ passed: true,
170
+ attempts: 0,
171
+ rolledBack: false,
172
+ snapshotId: null,
173
+ hotfixFiles: [],
174
+ lastOutput: check.output,
175
+ }
176
+ }
177
+
178
+ let attempts = 0
179
+ while (!check.passed && attempts < retries) {
180
+ if (signal?.aborted) break
181
+ attempts++
182
+
183
+ if (typeof onProgress === 'function') {
184
+ onProgress(`🔧 *Self-Healing Attempt ${attempts}/${retries}: Requesting hotfix patch from aggregator...*\n`)
185
+ }
186
+
187
+ const fileList = Array.isArray(promotedFiles)
188
+ ? promotedFiles.map((f) => (typeof f === 'string' ? f : f.relativePath || f.path)).filter(Boolean)
189
+ : []
190
+
191
+ const hotfixPrompt = [
192
+ `The promoted MoA changes failed the verification command \`${command}\` (exit code ${check.exitCode}).`,
193
+ `Original User Task: ${userPrompt}`,
194
+ `Promoted Files: ${fileList.join(', ') || 'workspace files'}`,
195
+ `\n### Compiler / Test Failure Output:\n\`\`\`\n${check.output}\n\`\`\``,
196
+ `\nProvide a minimal hotfix patch using markdown code blocks with \`path=<relative/path>\` header so the build and tests pass.`,
197
+ ].join('\n')
198
+
199
+ const llmRes = await callLlm({
200
+ provider: aggregatorSlot?.provider || 'default',
201
+ model: aggregatorSlot?.model || 'default',
202
+ messages: [
203
+ {
204
+ role: 'system',
205
+ content: 'You are an expert automated build healer. Fix compilation or unit test errors by outputting complete corrected files with ```lang path=relative/path.ext fences.',
206
+ },
207
+ { role: 'user', content: hotfixPrompt },
208
+ ],
209
+ temperature: 0.2,
210
+ maxTokens: 4096,
211
+ signal,
212
+ })
213
+
214
+ const patchText = typeof llmRes === 'string' ? llmRes : (llmRes?.content || llmRes?.text || '')
215
+ const extracted = extractFileBlocks(patchText)
216
+ if (extracted.length > 0) {
217
+ const applied = await applyHotfixFiles(cwd, extracted)
218
+ for (const f of applied) hotfixFiles.add(f)
219
+ }
220
+
221
+ check = await runVerificationCommand(cwd, command, { execFn, signal })
222
+ if (check.passed) {
223
+ if (typeof onProgress === 'function') {
224
+ onProgress(`✅ *Self-Healing Succeeded on attempt ${attempts}/${retries}!*\n`)
225
+ }
226
+ return {
227
+ enabled: true,
228
+ passed: true,
229
+ attempts,
230
+ rolledBack: false,
231
+ snapshotId: null,
232
+ hotfixFiles: [...hotfixFiles],
233
+ lastOutput: check.output,
234
+ }
235
+ }
236
+ }
237
+
238
+ // Verification still failed after maxRetries -> unconditional Time Machine rollback
239
+ const snapshotObj = checkpoint || promoteOpts?.checkpointSnapshot || (Array.isArray(promotedFiles) && promotedFiles._checkpoint) || null
240
+ if (typeof onProgress === 'function') {
241
+ onProgress(`⏪ *Self-Healing exhausted (${attempts}/${retries}). Rolling back workspace via dsh-time-machine...*\n`)
242
+ }
243
+
244
+ const rb = await rollbackToTimeMachineSnapshot(cwd, snapshotObj, {
245
+ ...promoteOpts,
246
+ rollbackFn,
247
+ })
248
+
249
+ return {
250
+ enabled: true,
251
+ passed: false,
252
+ attempts,
253
+ rolledBack: rb.rolledBack,
254
+ snapshotId: rb.snapshotId,
255
+ hotfixFiles: [...hotfixFiles],
256
+ lastOutput: check.output,
257
+ }
258
+ }
@@ -0,0 +1,187 @@
1
+ /**
2
+ * Multimodal Visual MoA Consensus via dsh-vision-bridge and dsh-live-canvas (#250)
3
+ * Renders UI/Frontend candidates (HTML, React, CSS, SVG) via dsh-live-canvas,
4
+ * captures headless screenshots, and evaluates layout/responsiveness via dsh-vision-bridge.
5
+ * Gracefully degrades to text evaluation when companion plugins are unavailable.
6
+ */
7
+
8
+ import { service } from './services.js'
9
+ import { logger } from './logger.js'
10
+ import { slotLabel } from './moa-prompts.js'
11
+
12
+ const UI_TASK_REGEX = /\b(html|css|svg|ui|frontend|react|vue|tailwind|layout|landing|page|component|button|modal|responsive|card|dashboard|widget)\b/i
13
+
14
+ /**
15
+ * Checks whether a prompt or candidate output involves UI/Frontend rendering.
16
+ */
17
+ export function isVisualUiTask(prompt = '', referenceOutputs = []) {
18
+ if (UI_TASK_REGEX.test(String(prompt || ''))) return true
19
+ for (const ref of referenceOutputs || []) {
20
+ const files = Array.isArray(ref?.files) ? ref.files : []
21
+ if (files.some((f) => /\.(html?|css|svg|jsx|tsx|vue)$/i.test(f.relativePath || f.path || ''))) {
22
+ return true
23
+ }
24
+ }
25
+ return false
26
+ }
27
+
28
+ /**
29
+ * Extracts renderable HTML/SVG preview markup from a candidate output.
30
+ */
31
+ export function extractRenderableMarkup(candidate = {}) {
32
+ const files = Array.isArray(candidate.files) ? candidate.files : []
33
+ const htmlFile = files.find((f) => /\.(html?|svg)$/i.test(f.relativePath || f.path || ''))
34
+ if (htmlFile?.content) {
35
+ return { title: htmlFile.relativePath || 'index.html', content: htmlFile.content }
36
+ }
37
+
38
+ const cssFile = files.find((f) => /\.css$/i.test(f.relativePath || f.path || ''))
39
+ const jsFile = files.find((f) => /\.(jsx?|tsx?)$/i.test(f.relativePath || f.path || ''))
40
+ if (cssFile || jsFile) {
41
+ const html = [
42
+ '<!DOCTYPE html><html><head><meta charset="utf-8">',
43
+ cssFile ? `<style>${cssFile.content}</style>` : '',
44
+ '</head><body><div id="app">UI Preview</div>',
45
+ '</body></html>',
46
+ ].join('\n')
47
+ return { title: 'synthesized-preview.html', content: html }
48
+ }
49
+
50
+ const text = String(candidate.text || '')
51
+ const fenceMatch = /```(?:html|svg)\s*([\s\S]*?)```/i.exec(text)
52
+ if (fenceMatch) {
53
+ return { title: 'inline-preview.html', content: fenceMatch[1].trim() }
54
+ }
55
+ return null
56
+ }
57
+
58
+ /**
59
+ * Evaluates candidates visually using dsh-live-canvas and dsh-vision-bridge.
60
+ * Returns { degraded: true } if companion services are absent so the caller falls back cleanly.
61
+ *
62
+ * @param {object} params
63
+ * @returns {Promise<{
64
+ * degraded: boolean,
65
+ * reason?: string,
66
+ * visualScores: Array<{ candidateIndex: number, label: string, visualScore: number, critique: string, previewUrl: string | null }>,
67
+ * directive: string
68
+ * }>}
69
+ */
70
+ export async function runVisualMoaConsensus({
71
+ ctx = null,
72
+ liveCanvas = null,
73
+ visionBridge = null,
74
+ userPrompt = '',
75
+ referenceOutputs = [],
76
+ onProgress = null,
77
+ }) {
78
+ const resolvedCanvas = liveCanvas || service(ctx, 'liveCanvas')
79
+ const resolvedVision = visionBridge || service(ctx, 'visionBridge')
80
+
81
+ if (!resolvedCanvas && !resolvedVision) {
82
+ return {
83
+ degraded: true,
84
+ reason: 'Companion plugins dsh-live-canvas and dsh-vision-bridge are not active; using standard text evaluation.',
85
+ visualScores: [],
86
+ directive: '',
87
+ }
88
+ }
89
+
90
+ if (typeof onProgress === 'function') {
91
+ onProgress('👁️ *Visual MoA Consensus: Rendering UI candidates in Live Canvas & evaluating via Vision Bridge...*\n')
92
+ }
93
+
94
+ const visualScores = []
95
+ for (let i = 0; i < referenceOutputs.length; i++) {
96
+ const ref = referenceOutputs[i]
97
+ if (!ref || ref.ok === false) continue
98
+ const idx = ref.index ?? (i + 1)
99
+ const label = ref.label || slotLabel(ref.slot || ref)
100
+ const markup = extractRenderableMarkup(ref)
101
+ if (!markup) continue
102
+
103
+ let previewUrl = null
104
+ let screenshot = null
105
+
106
+ if (resolvedCanvas) {
107
+ try {
108
+ if (typeof resolvedCanvas.createPreviewFromContent === 'function') {
109
+ const prev = await resolvedCanvas.createPreviewFromContent({
110
+ content: markup.content,
111
+ title: `Candidate #${idx} (${label}) - ${markup.title}`,
112
+ })
113
+ previewUrl = prev?.previewUrl || null
114
+ screenshot = prev?.screenshot || prev?.imageBase64 || null
115
+ }
116
+ if (!screenshot && typeof resolvedCanvas.captureScreenshot === 'function') {
117
+ screenshot = await resolvedCanvas.captureScreenshot({
118
+ content: markup.content,
119
+ previewUrl,
120
+ })
121
+ }
122
+ } catch (err) {
123
+ logger.warn(`[dsh-moa:visual] Live Canvas preview failed for candidate #${idx}:`, err?.message || err)
124
+ }
125
+ }
126
+
127
+ let visualScore = 7.5
128
+ let critique = 'Rendered cleanly in Live Canvas preview.'
129
+
130
+ if (resolvedVision) {
131
+ try {
132
+ const evalFn = resolvedVision.evaluateVisualLayout || resolvedVision.analyzeImage || resolvedVision.inspectUi
133
+ if (typeof evalFn === 'function') {
134
+ const vRes = await evalFn.call(resolvedVision, {
135
+ prompt: userPrompt,
136
+ candidateIndex: idx,
137
+ html: markup.content,
138
+ previewUrl,
139
+ screenshot,
140
+ })
141
+ if (vRes) {
142
+ if (Number.isFinite(Number(vRes.score))) {
143
+ visualScore = Math.max(1, Math.min(10, Number(vRes.score)))
144
+ }
145
+ if (vRes.critique || vRes.summary || vRes.feedback) {
146
+ critique = String(vRes.critique || vRes.summary || vRes.feedback).trim()
147
+ }
148
+ }
149
+ }
150
+ } catch (err) {
151
+ logger.warn(`[dsh-moa:visual] Vision Bridge evaluation failed for candidate #${idx}:`, err?.message || err)
152
+ }
153
+ }
154
+
155
+ visualScores.push({
156
+ candidateIndex: idx,
157
+ label,
158
+ visualScore: Number(visualScore.toFixed(1)),
159
+ critique,
160
+ previewUrl,
161
+ })
162
+ }
163
+
164
+ if (visualScores.length === 0) {
165
+ return {
166
+ degraded: true,
167
+ reason: 'No renderable HTML/CSS/SVG artifacts found in candidates.',
168
+ visualScores: [],
169
+ directive: '',
170
+ }
171
+ }
172
+
173
+ const lines = [
174
+ '### 👁️ Multimodal Visual MoA Consensus (`dsh-live-canvas` + `dsh-vision-bridge`):',
175
+ 'The following UI candidates were rendered and visually inspected for layout, responsiveness, and design fidelity:',
176
+ ...visualScores.map(
177
+ (v) => `- **Candidate #${v.candidateIndex} (${v.label})**: Visual Score **${v.visualScore}/10** — ${v.critique}${v.previewUrl ? ` ([Live Preview](${v.previewUrl}))` : ''}`
178
+ ),
179
+ 'Prioritize candidates with higher visual layout fidelity and zero rendering defects.',
180
+ ]
181
+
182
+ return {
183
+ degraded: false,
184
+ visualScores,
185
+ directive: lines.join('\n'),
186
+ }
187
+ }
@@ -27,6 +27,30 @@ export const COMPANION_PLUGINS = [
27
27
  url: 'https://github.com/goodandready/dsh-time-machine',
28
28
  description: 'Pre-promotion shadow Git checkpoints and rollback points',
29
29
  },
30
+ {
31
+ id: 'dsh-key-limits',
32
+ name: 'DSH Key Limits',
33
+ url: 'https://github.com/GooDAnDReaDY/dsh-key-limits',
34
+ description: 'Provider API key rate-limit and quota telemetry',
35
+ },
36
+ {
37
+ id: 'dsh-subscriptions',
38
+ name: 'DSH Subscriptions',
39
+ url: 'https://github.com/GooDAnDReaDY/dsh-subscriptions',
40
+ description: 'Subscription tier and quota tracking for LLM providers',
41
+ },
42
+ {
43
+ id: 'dsh-vision-bridge',
44
+ name: 'DSH Vision Bridge',
45
+ url: 'https://github.com/GooDAnDReaDY/dsh-vision-bridge',
46
+ description: 'Multimodal VLM visual evaluation for UI/Frontend artifacts',
47
+ },
48
+ {
49
+ id: 'dsh-live-canvas',
50
+ name: 'DSH Live Canvas',
51
+ url: 'https://github.com/GooDAnDReaDY/dsh-live-canvas',
52
+ description: 'Isolated live HTML/CSS/SVG preview environment and screenshot capture',
53
+ },
30
54
  ]
31
55
 
32
56
  /**
@@ -1,3 +1,4 @@
1
+ import { balanceCandidateQuotas } from '../moa-quota-balancer.js'
1
2
  import { collectProjectContext, formatProjectContext, isRefinementTask } from '../workspace/scanner.js'
2
3
  import { applyBudgetGuardrails, createBudgetAbortPayload } from '../moa-budget.js'
3
4
  import { extractPriorTurnBaseline, pruneMultiTurnMessages } from '../moa-context.js'
@@ -13,6 +14,7 @@ export async function stageProjectContext({
13
14
  maxTokens,
14
15
  startTime,
15
16
  onProgress,
17
+ ctx = null,
16
18
  }) {
17
19
  const rawRefs = (Array.isArray(preset?.reference_models) && preset.reference_models.length > 0)
18
20
  ? preset.reference_models
@@ -79,7 +81,23 @@ export async function stageProjectContext({
79
81
  return { ok: false, abortPayload }
80
82
  }
81
83
 
82
- const effectiveReferences = budgetCheck.references
84
+ let effectiveReferences = budgetCheck.references
85
+ let quotaSubstitutions = []
86
+ if (preset?.quota_aware_balancing_enabled) {
87
+ const fallbackPool = Array.isArray(preset?.local_fallback_models) && preset.local_fallback_models.length > 0
88
+ ? preset.local_fallback_models
89
+ : undefined
90
+ const bal = await balanceCandidateQuotas(effectiveReferences, {
91
+ ctx,
92
+ aggregator: primaryJudge,
93
+ fallbackPool,
94
+ })
95
+ effectiveReferences = bal.balancedReferences
96
+ quotaSubstitutions = bal.substitutions
97
+ if (quotaSubstitutions.length > 0 && typeof onProgress === "function") {
98
+ onProgress(`⚖️ *Quota-Aware Balancing: Replaced ${quotaSubstitutions.length} low-quota slot(s) to preserve aggregator budget.*\n`)
99
+ }
100
+ }
83
101
  if (effectiveReferences.length === 1 && !preset?.curator_synthesis) {
84
102
  isFastMode = true
85
103
  }
@@ -103,7 +121,7 @@ export async function stageProjectContext({
103
121
  isStreamAggregator, isQuorumEnabled, gracePeriodSec, candidateRetries,
104
122
  refTimeoutSec, aggTimeoutSec, isBlindEvaluation, isPeerCritiqueEnabled,
105
123
  allowCandidateOverride, collectedCtx, isRefinement, projectContext,
106
- effectiveReferences, priorTurnBaseline, candidateSystemPrompt,
124
+ effectiveReferences, quotaSubstitutions, priorTurnBaseline, candidateSystemPrompt,
107
125
  needsQuestions, enrichedMessages,
108
126
  }
109
127
  }
@@ -1,3 +1,5 @@
1
+ import { classifyEloCategory } from '../moa-elo.js'
2
+ import { runSelfHealingLoop } from '../moa-self-healing.js'
1
3
  import { logger } from '../logger.js'
2
4
  import { parseWinnerIndex, parseRecommendedAssembler } from '../moa-parser.js'
3
5
  import { extractFileBlocks, writeCandidateWorkspace, promoteCandidateWorkspace, cleanMoaWorkspaces } from '../workspace/fs.js'
@@ -33,6 +35,10 @@ export async function stagePromotion({
33
35
  historyFilePath,
34
36
  collectedCtx,
35
37
  ctx = null,
38
+ callLlm = null,
39
+ onProgress = null,
40
+ execFn = null,
41
+ visualResult = null,
36
42
  }) {
37
43
  const winningIndex = (multiJudgeResult?.consensus && multiJudgeResult?.winningCandidateIndex)
38
44
  ? multiJudgeResult.winningCandidateIndex
@@ -69,6 +75,24 @@ export async function stagePromotion({
69
75
  throw promoteErr
70
76
  }
71
77
 
78
+ let selfHealingResult = null
79
+ if (preset?.self_healing_enabled && promotedFiles.length > 0 && typeof callLlm === "function") {
80
+ selfHealingResult = await runSelfHealingLoop({
81
+ cwd,
82
+ command: preset.self_healing_command || "npm test",
83
+ maxRetries: preset.self_healing_max_retries ?? 2,
84
+ aggregatorSlot: chosenJudge,
85
+ callLlm,
86
+ userPrompt,
87
+ promotedFiles,
88
+ checkpoint: promotedFiles.checkpoint || null,
89
+ promoteOpts,
90
+ onProgress,
91
+ signal,
92
+ execFn,
93
+ })
94
+ }
95
+
72
96
  const livePreview = await createPromotedPreview(liveCanvas, cwd, promotedFiles, referenceOutputs, { winningIndex, synthesizedFiles })
73
97
  const durationMs = Date.now() - startTime
74
98
  const usageSummary = summarizeMoAUsage(referenceOutputs, aggUsage, multiJudgeResult?.totalUsage)
@@ -100,7 +124,7 @@ export async function stagePromotion({
100
124
 
101
125
  try {
102
126
  await recordMoaRunAsync({
103
- id: runId, cwd, prompt: userPrompt, preset: preset?.name || 'default', isRefinement,
127
+ id: runId, cwd, prompt: userPrompt, category: classifyEloCategory(userPrompt), preset: preset?.name || 'default', isRefinement,
104
128
  candidates: candidatesForHistory(referenceOutputs),
105
129
  aggregator: { provider: chosenJudge.provider, model: chosenJudge.model, usage: aggUsage, costUsd: aggUsage.costUsd },
106
130
  winnerIndex: winningIndex, winnerModel, promotedFiles, benchmarkReport, consensus: multiJudgeResult,
@@ -125,7 +149,7 @@ export async function stagePromotion({
125
149
  kind: 'synthesis', content: synthesizedText, aggregator: isFastMode ? 'Fast Mode (Direct)' : finalAggLabel,
126
150
  references: referenceOutputs, presetName: preset?.name || 'default', isRefinement, isFastMode, isCuratorSynthesis,
127
151
  recommendedAssembler, winningIndex, winnerModel, promotedFiles, runId, allowCandidateOverride,
128
- benchmarkReport, consensus: multiJudgeResult, fileScores, ...(livePreview ? { liveCanvas: livePreview } : {}),
152
+ benchmarkReport, consensus: multiJudgeResult, fileScores, ...(selfHealingResult ? { selfHealing: selfHealingResult } : {}), ...(visualResult ? { visualConsensus: visualResult } : {}), ...(livePreview ? { liveCanvas: livePreview } : {}),
129
153
  usage: usageSummary, skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs,
130
154
  }
131
155
  }
@@ -1,3 +1,4 @@
1
+ import { runVisualMoaConsensus } from '../moa-visual-consensus.js'
1
2
  import crypto from 'node:crypto'
2
3
  import { isTestEnvironment } from '../history.js'
3
4
  import { cleanMoaWorkspaces } from '../workspace/fs.js'
@@ -40,7 +41,7 @@ export async function runMoAPipeline({
40
41
 
41
42
  // Stage 1: Context & Configuration
42
43
  const ctxStage = await stageProjectContext({
43
- userPrompt, messages, preset, cwd, prices, maxTokens: preset?.max_tokens, startTime, onProgress,
44
+ userPrompt, messages, preset, cwd, prices, maxTokens: preset?.max_tokens, startTime, onProgress, ctx,
44
45
  })
45
46
  if (!ctxStage.ok) return ctxStage.abortPayload
46
47
 
@@ -157,9 +158,20 @@ export async function runMoAPipeline({
157
158
  const compositeMergeDirective = (preset?.composite_merge_enabled && successfulRefs.length > 1)
158
159
  ? buildCompositeBlockDirectives(successfulRefs, { blind: Boolean(preset.blind_evaluation) })
159
160
  : null
160
- const fileScoringDirective = (preset?.interactive_workbench_enabled)
161
+ let visualResult = null
162
+ if (preset?.visual_moa_enabled) {
163
+ visualResult = await runVisualMoaConsensus({
164
+ ctx,
165
+ liveCanvas,
166
+ userPrompt,
167
+ referenceOutputs,
168
+ onProgress,
169
+ })
170
+ }
171
+ const rawFileDirective = (preset?.interactive_workbench_enabled)
161
172
  ? formatFileScoresDirective()
162
173
  : null
174
+ const fileScoringDirective = [rawFileDirective, visualResult?.directive].filter(Boolean).join("\n\n") || null
163
175
 
164
176
  // Stage 7: Synthesis Phase
165
177
  const synthRes = await stageSynthesis({
@@ -189,7 +201,7 @@ export async function runMoAPipeline({
189
201
  promoteOpts, cwd, runId, signal, liveCanvas, startTime, chosenJudge, aggUsage,
190
202
  userPrompt, preset, isRefinement: ctxStage.isRefinement, isFastMode: ctxStage.isFastMode,
191
203
  effectiveReferences: ctxStage.effectiveReferences, referenceModels: ctxStage.referenceModels,
192
- historyFilePath, collectedCtx: ctxStage.collectedCtx, ctx,
204
+ historyFilePath, collectedCtx: ctxStage.collectedCtx, ctx, callLlm, onProgress, execFn, visualResult,
193
205
  })
194
206
  if (!promRes.ok) {
195
207
  if (promRes.abort) return { error: promRes.error }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@goodandready/dsh-moa",
3
- "version": "0.2.68",
3
+ "version": "0.2.70",
4
4
  "description": "Mixture of Agents (MoA) plugin for DeepSeek Harness with /moa slash command",
5
5
  "type": "module",
6
6
  "main": "lib/index.js",