@goodandready/dsh-moa 0.2.68 → 0.2.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +108 -0
- package/lib/history.js +21 -6
- package/lib/moa-elo.js +189 -0
- package/lib/moa-quota-balancer.js +188 -0
- package/lib/moa-router.js +27 -9
- package/lib/moa-schema.js +12 -0
- package/lib/moa-self-healing.js +258 -0
- package/lib/moa-visual-consensus.js +187 -0
- package/lib/moa-workbench.js +24 -0
- package/lib/pipeline/context.js +20 -2
- package/lib/pipeline/promotion.js +26 -2
- package/lib/pipeline/runner.js +15 -3
- package/package.json +1 -1
package/lib/client.js
CHANGED
|
@@ -2017,6 +2017,7 @@ window.__ModuleLoader__.load({
|
|
|
2017
2017
|
t("aggregator.report_desc") || "Generate downloadable Markdown & JSON reports detailing candidate latency, agreement, test gate, and cost breakdown."
|
|
2018
2018
|
)
|
|
2019
2019
|
),
|
|
2020
|
+
// --- src/client/85-card-aggregator-ext.js ---
|
|
2020
2021
|
/* Interactive Workbench & File Scoring (#251) */
|
|
2021
2022
|
React.createElement(
|
|
2022
2023
|
"div",
|
|
@@ -2054,6 +2055,111 @@ window.__ModuleLoader__.load({
|
|
|
2054
2055
|
"(" + (t("aggregator.workbench_companion_note") || "safe shadow Git checkpoints before file promotion") + ")"
|
|
2055
2056
|
)
|
|
2056
2057
|
),
|
|
2058
|
+
/* Multimodal Visual MoA Consensus (#250) */
|
|
2059
|
+
React.createElement(
|
|
2060
|
+
"div",
|
|
2061
|
+
{ style: { display: "flex", flexDirection: "column", gap: 6, padding: "8px 12px", background: "var(--dsw-alias-bg-layer-2)", borderRadius: 8, border: "1px solid var(--dsw-alias-border-l2)", marginTop: 8 } },
|
|
2062
|
+
React.createElement(
|
|
2063
|
+
"label",
|
|
2064
|
+
{ style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer", fontSize: 13, fontWeight: 500 } },
|
|
2065
|
+
React.createElement("input", {
|
|
2066
|
+
type: "checkbox",
|
|
2067
|
+
checked: Boolean(currentPreset.visual_moa_enabled),
|
|
2068
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, visual_moa_enabled: e.target.checked })),
|
|
2069
|
+
style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
|
|
2070
|
+
}),
|
|
2071
|
+
t("aggregator.visual_moa_title") || "\u{1F441}\uFE0F Multimodal Visual MoA Consensus"
|
|
2072
|
+
),
|
|
2073
|
+
React.createElement(
|
|
2074
|
+
"div",
|
|
2075
|
+
{ style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
|
|
2076
|
+
t("aggregator.visual_moa_desc") || "Render UI candidates in Live Canvas and evaluate visual layout fidelity via Vision Bridge VLM."
|
|
2077
|
+
),
|
|
2078
|
+
React.createElement(
|
|
2079
|
+
"div",
|
|
2080
|
+
{ style: { fontSize: 11, color: "var(--dsw-alias-label-tertiary)", marginLeft: 22, marginTop: 2, display: "flex", alignItems: "center", gap: 6, flexWrap: "wrap" } },
|
|
2081
|
+
"\u{1F517} Companion Plugins:",
|
|
2082
|
+
React.createElement(
|
|
2083
|
+
"a",
|
|
2084
|
+
{ href: "https://github.com/GooDAnDReaDY/dsh-vision-bridge", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
|
|
2085
|
+
"@goodandready/dsh-vision-bridge"
|
|
2086
|
+
),
|
|
2087
|
+
"/",
|
|
2088
|
+
React.createElement(
|
|
2089
|
+
"a",
|
|
2090
|
+
{ href: "https://github.com/GooDAnDReaDY/dsh-live-canvas", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
|
|
2091
|
+
"@goodandready/dsh-live-canvas"
|
|
2092
|
+
)
|
|
2093
|
+
)
|
|
2094
|
+
),
|
|
2095
|
+
/* Quota-Aware Provider Balancing (#253) */
|
|
2096
|
+
React.createElement(
|
|
2097
|
+
"div",
|
|
2098
|
+
{ style: { display: "flex", flexDirection: "column", gap: 6, padding: "8px 12px", background: "var(--dsw-alias-bg-layer-2)", borderRadius: 8, border: "1px solid var(--dsw-alias-border-l2)", marginTop: 8 } },
|
|
2099
|
+
React.createElement(
|
|
2100
|
+
"label",
|
|
2101
|
+
{ style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer", fontSize: 13, fontWeight: 500 } },
|
|
2102
|
+
React.createElement("input", {
|
|
2103
|
+
type: "checkbox",
|
|
2104
|
+
checked: Boolean(currentPreset.quota_aware_balancing_enabled),
|
|
2105
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, quota_aware_balancing_enabled: e.target.checked })),
|
|
2106
|
+
style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
|
|
2107
|
+
}),
|
|
2108
|
+
t("aggregator.quota_balancing_title") || "\u2696\uFE0F Quota-Aware Provider Balancing"
|
|
2109
|
+
),
|
|
2110
|
+
React.createElement(
|
|
2111
|
+
"div",
|
|
2112
|
+
{ style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
|
|
2113
|
+
t("aggregator.quota_balancing_desc") || "Automatically substitute low-quota candidate models in Layer 1 to preserve rate limits for the final aggregator."
|
|
2114
|
+
),
|
|
2115
|
+
React.createElement(
|
|
2116
|
+
"div",
|
|
2117
|
+
{ style: { fontSize: 11, color: "var(--dsw-alias-label-tertiary)", marginLeft: 22, marginTop: 2, display: "flex", alignItems: "center", gap: 6, flexWrap: "wrap" } },
|
|
2118
|
+
"\u{1F517} Companion Plugins:",
|
|
2119
|
+
React.createElement(
|
|
2120
|
+
"a",
|
|
2121
|
+
{ href: "https://github.com/GooDAnDReaDY/dsh-key-limits", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
|
|
2122
|
+
"@goodandready/dsh-key-limits"
|
|
2123
|
+
),
|
|
2124
|
+
"/",
|
|
2125
|
+
React.createElement(
|
|
2126
|
+
"a",
|
|
2127
|
+
{ href: "https://github.com/GooDAnDReaDY/dsh-subscriptions", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
|
|
2128
|
+
"@goodandready/dsh-subscriptions"
|
|
2129
|
+
)
|
|
2130
|
+
)
|
|
2131
|
+
),
|
|
2132
|
+
/* Self-Healing Post-Promotion Build Loop (#252) */
|
|
2133
|
+
React.createElement(
|
|
2134
|
+
"div",
|
|
2135
|
+
{ style: { display: "flex", flexDirection: "column", gap: 6, padding: "8px 12px", background: "var(--dsw-alias-bg-layer-2)", borderRadius: 8, border: "1px solid var(--dsw-alias-border-l2)", marginTop: 8 } },
|
|
2136
|
+
React.createElement(
|
|
2137
|
+
"label",
|
|
2138
|
+
{ style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer", fontSize: 13, fontWeight: 500 } },
|
|
2139
|
+
React.createElement("input", {
|
|
2140
|
+
type: "checkbox",
|
|
2141
|
+
checked: Boolean(currentPreset.self_healing_enabled),
|
|
2142
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, self_healing_enabled: e.target.checked })),
|
|
2143
|
+
style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
|
|
2144
|
+
}),
|
|
2145
|
+
t("aggregator.self_healing_title") || "\u{1FA7A} Self-Healing Post-Promotion Build Loop"
|
|
2146
|
+
),
|
|
2147
|
+
React.createElement(
|
|
2148
|
+
"div",
|
|
2149
|
+
{ style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
|
|
2150
|
+
t("aggregator.self_healing_desc") || "Verify promoted files with a test command, request hotfix patches from the aggregator on failure, or auto-rollback to the Time Machine checkpoint."
|
|
2151
|
+
),
|
|
2152
|
+
React.createElement(
|
|
2153
|
+
"div",
|
|
2154
|
+
{ style: { fontSize: 11, color: "var(--dsw-alias-label-tertiary)", marginLeft: 22, marginTop: 2, display: "flex", alignItems: "center", gap: 6 } },
|
|
2155
|
+
"\u{1F517} Companion Plugin:",
|
|
2156
|
+
React.createElement(
|
|
2157
|
+
"a",
|
|
2158
|
+
{ href: "https://github.com/GooDAnDReaDY/dsh-time-machine", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
|
|
2159
|
+
"@goodandready/dsh-time-machine"
|
|
2160
|
+
)
|
|
2161
|
+
)
|
|
2162
|
+
),
|
|
2057
2163
|
/* Graceful Degradation & Local Fallback (#175, #176) */
|
|
2058
2164
|
React.createElement(
|
|
2059
2165
|
"div",
|
|
@@ -2511,6 +2617,7 @@ window.__ModuleLoader__.load({
|
|
|
2511
2617
|
React.createElement("th", { style: { padding: "6px 8px" } }, t("leaderboard.runs")),
|
|
2512
2618
|
React.createElement("th", { style: { padding: "6px 8px" } }, t("leaderboard.wins")),
|
|
2513
2619
|
React.createElement("th", { style: { padding: "6px 8px" } }, t("leaderboard.winrate")),
|
|
2620
|
+
React.createElement("th", { style: { padding: "6px 8px" } }, "Elo"),
|
|
2514
2621
|
React.createElement("th", { style: { padding: "6px 10px" } }, t("leaderboard.cost"))
|
|
2515
2622
|
)
|
|
2516
2623
|
),
|
|
@@ -2525,6 +2632,7 @@ window.__ModuleLoader__.load({
|
|
|
2525
2632
|
React.createElement("td", { style: { padding: "6px 8px", color: "var(--dsw-alias-label-secondary)" } }, String(m.runs)),
|
|
2526
2633
|
React.createElement("td", { style: { padding: "6px 8px", color: "var(--dsw-alias-state-success-primary)" } }, String(m.wins)),
|
|
2527
2634
|
React.createElement("td", { style: { padding: "6px 8px", fontWeight: 600 } }, (m.winRate ?? 0) + "%"),
|
|
2635
|
+
React.createElement("td", { style: { padding: "6px 8px", fontWeight: 600, color: "var(--dsw-alias-state-brand-primary)" } }, String(m.elo ?? 1500)),
|
|
2528
2636
|
React.createElement("td", { style: { padding: "6px 10px", color: "var(--dsw-alias-label-secondary)" } }, m.avgCostUsd > 0 ? "$" + m.avgCostUsd.toFixed(4) : "Free")
|
|
2529
2637
|
)
|
|
2530
2638
|
)
|
package/lib/history.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { classifyEloCategory, computeEloFromRuns, INITIAL_ELO } from './moa-elo.js'
|
|
1
2
|
import { logger } from './logger.js'
|
|
2
3
|
/**
|
|
3
4
|
* MoA Run History & Analytics Module
|
|
@@ -355,6 +356,7 @@ export function getMoaLeaderboard(filePath = getDefaultHistoryFile(), options =
|
|
|
355
356
|
_historyCache.totalCount = totalRuns
|
|
356
357
|
|
|
357
358
|
const modelStats = new Map()
|
|
359
|
+
const parsedRuns = []
|
|
358
360
|
let filteredRuns = 0
|
|
359
361
|
for (const line of lines) {
|
|
360
362
|
try {
|
|
@@ -362,6 +364,7 @@ export function getMoaLeaderboard(filePath = getDefaultHistoryFile(), options =
|
|
|
362
364
|
if (!item) continue
|
|
363
365
|
if (presetFilter && presetFilter !== 'all' && item.preset !== presetFilter) continue
|
|
364
366
|
filteredRuns++
|
|
367
|
+
parsedRuns.push(item)
|
|
365
368
|
|
|
366
369
|
const rawWinner = item.winnerModel ? String(item.winnerModel).trim() : null
|
|
367
370
|
const cleanWinner = rawWinner ? rawWinner.replace(/\s*\[fallback for .*\]$/i, '').trim() : null
|
|
@@ -415,13 +418,25 @@ export function getMoaLeaderboard(filePath = getDefaultHistoryFile(), options =
|
|
|
415
418
|
}
|
|
416
419
|
}
|
|
417
420
|
|
|
421
|
+
const { globalRatings, categoryRatings } = computeEloFromRuns(parsedRuns)
|
|
418
422
|
const models = Array.from(modelStats.values())
|
|
419
|
-
.map((st) =>
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
423
|
+
.map((st) => {
|
|
424
|
+
const elo = Math.round(globalRatings.get(st.modelKey) ?? INITIAL_ELO)
|
|
425
|
+
const categoryElo = {}
|
|
426
|
+
for (const [cat, cMap] of categoryRatings.entries()) {
|
|
427
|
+
if (cMap.has(st.modelKey)) {
|
|
428
|
+
categoryElo[cat] = Math.round(cMap.get(st.modelKey))
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return {
|
|
432
|
+
...st,
|
|
433
|
+
elo,
|
|
434
|
+
categoryElo,
|
|
435
|
+
winRate: st.runs > 0 ? Number(((st.wins / st.runs) * 100).toFixed(1)) : 0,
|
|
436
|
+
avgTokens: st.runs > 0 ? Math.round(st.totalTokens / st.runs) : 0,
|
|
437
|
+
avgCostUsd: st.runs > 0 ? Number((st.totalCostUsd / st.runs).toFixed(4)) : 0,
|
|
438
|
+
}
|
|
439
|
+
})
|
|
425
440
|
.sort((a, b) => b.wins - a.wins || b.winRate - a.winRate)
|
|
426
441
|
|
|
427
442
|
const result = {
|
package/lib/moa-elo.js
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Empirical Elo Rating and Active Learning Leaderboard by Task Category (#249)
|
|
3
|
+
* Calculates pairwise Elo ratings (K-factor = 32, baseline = 1500) segmented by
|
|
4
|
+
* task category (frontend, backend, refactoring, testing, sql, general),
|
|
5
|
+
* and exposes category leaders to Smart Routing.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export const INITIAL_ELO = 1500
|
|
9
|
+
export const DEFAULT_K_FACTOR = 32
|
|
10
|
+
|
|
11
|
+
const CATEGORY_PATTERNS = [
|
|
12
|
+
{ category: 'sql', regex: /\b(sql|postgres|sqlite|mysql|query|database|schema|migration|prisma|index|table)\b/i },
|
|
13
|
+
{ category: 'frontend', regex: /\b(frontend|ui|css|html|react|vue|tailwind|component|button|layout|modal|svg|dom)\b/i },
|
|
14
|
+
{ category: 'testing', regex: /\b(test|unit\s*test|e2e|vitest|jest|playwright|assert|coverage|mock|tdd)\b/i },
|
|
15
|
+
{ category: 'refactoring', regex: /\b(refactor|clean|simplify|dedup|modular|rename|extract|decompose|yagni)\b/i },
|
|
16
|
+
{ category: 'backend', regex: /\b(backend|api|endpoint|server|route|auth|jwt|middleware|express|http|rest|webhook|stream)\b/i },
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Classifies a prompt into one of the standard Elo leaderboard categories.
|
|
21
|
+
*
|
|
22
|
+
* @param {string} prompt
|
|
23
|
+
* @returns {'frontend' | 'backend' | 'refactoring' | 'testing' | 'sql' | 'general'}
|
|
24
|
+
*/
|
|
25
|
+
export function classifyEloCategory(prompt = '') {
|
|
26
|
+
const text = String(prompt || '').trim()
|
|
27
|
+
if (!text) return 'general'
|
|
28
|
+
for (const rule of CATEGORY_PATTERNS) {
|
|
29
|
+
if (rule.regex.test(text)) return rule.category
|
|
30
|
+
}
|
|
31
|
+
return 'general'
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Calculates expected win probability of player A against player B.
|
|
36
|
+
*/
|
|
37
|
+
export function expectedEloScore(ratingA, ratingB) {
|
|
38
|
+
return 1 / (1 + Math.pow(10, (ratingB - ratingA) / 400))
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Updates pairwise Elo ratings when winnerKey beats all other participantKeys in a round.
|
|
43
|
+
*
|
|
44
|
+
* @param {Map<string, number>} ratingMap
|
|
45
|
+
* @param {string} winnerKey
|
|
46
|
+
* @param {string[]} participantKeys
|
|
47
|
+
* @param {number} [kFactor=32]
|
|
48
|
+
*/
|
|
49
|
+
export function applyRoundEloUpdate(ratingMap, winnerKey, participantKeys = [], kFactor = DEFAULT_K_FACTOR) {
|
|
50
|
+
if (!winnerKey || participantKeys.length < 2) return
|
|
51
|
+
const opponents = participantKeys.filter((k) => k && k !== winnerKey)
|
|
52
|
+
if (opponents.length === 0) return
|
|
53
|
+
|
|
54
|
+
// Normalize K-factor across the number of pairwise comparisons in the multi-candidate round
|
|
55
|
+
const effectiveK = kFactor / opponents.length
|
|
56
|
+
let winnerRating = ratingMap.get(winnerKey) ?? INITIAL_ELO
|
|
57
|
+
let winnerDelta = 0
|
|
58
|
+
|
|
59
|
+
for (const oppKey of opponents) {
|
|
60
|
+
const oppRating = ratingMap.get(oppKey) ?? INITIAL_ELO
|
|
61
|
+
const expWin = expectedEloScore(winnerRating, oppRating)
|
|
62
|
+
const expOpp = 1 - expWin
|
|
63
|
+
|
|
64
|
+
winnerDelta += effectiveK * (1 - expWin)
|
|
65
|
+
const nextOpp = oppRating + effectiveK * (0 - expOpp)
|
|
66
|
+
ratingMap.set(oppKey, nextOpp)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
ratingMap.set(winnerKey, winnerRating + winnerDelta)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Computes global and per-category Elo ratings across a chronological list of MoA runs.
|
|
74
|
+
*
|
|
75
|
+
* @param {Array<object>} runs - Array of history run objects (oldest to newest)
|
|
76
|
+
* @param {number} [kFactor=32]
|
|
77
|
+
* @returns {{
|
|
78
|
+
* globalRatings: Map<string, number>,
|
|
79
|
+
* categoryRatings: Map<string, Map<string, number>>
|
|
80
|
+
* }}
|
|
81
|
+
*/
|
|
82
|
+
export function computeEloFromRuns(runs = [], kFactor = DEFAULT_K_FACTOR) {
|
|
83
|
+
const globalRatings = new Map()
|
|
84
|
+
const categoryRatings = new Map()
|
|
85
|
+
|
|
86
|
+
for (const run of runs) {
|
|
87
|
+
if (!run || !Array.isArray(run.candidates) || run.candidates.length === 0) continue
|
|
88
|
+
|
|
89
|
+
const category = run.category || classifyEloCategory(run.prompt || '')
|
|
90
|
+
if (!categoryRatings.has(category)) {
|
|
91
|
+
categoryRatings.set(category, new Map())
|
|
92
|
+
}
|
|
93
|
+
const catMap = categoryRatings.get(category)
|
|
94
|
+
|
|
95
|
+
const participantKeys = []
|
|
96
|
+
let winnerKey = null
|
|
97
|
+
|
|
98
|
+
const rawWinner = run.winnerModel ? String(run.winnerModel).trim() : null
|
|
99
|
+
const cleanWinner = rawWinner ? rawWinner.replace(/\s*\[fallback for .*\]$/i, '').trim() : null
|
|
100
|
+
const normWinner = cleanWinner && cleanWinner !== 'none' && cleanWinner !== 'unknown'
|
|
101
|
+
? cleanWinner.replace(/[\/:]/g, ':').toLowerCase()
|
|
102
|
+
: null
|
|
103
|
+
|
|
104
|
+
for (let idx = 0; idx < run.candidates.length; idx++) {
|
|
105
|
+
const cand = run.candidates[idx]
|
|
106
|
+
const key = cand.provider && cand.model ? `${cand.provider}:${cand.model}` : (cand.model || 'unknown')
|
|
107
|
+
participantKeys.push(key)
|
|
108
|
+
if (!globalRatings.has(key)) globalRatings.set(key, INITIAL_ELO)
|
|
109
|
+
if (!catMap.has(key)) catMap.set(key, INITIAL_ELO)
|
|
110
|
+
|
|
111
|
+
if (!winnerKey) {
|
|
112
|
+
if (normWinner) {
|
|
113
|
+
const normKey = key.replace(/[\/:]/g, ':').toLowerCase()
|
|
114
|
+
const normModel = String(cand.model || '').replace(/[\/:]/g, ':').toLowerCase()
|
|
115
|
+
if (normWinner === normKey || normWinner === normModel || normWinner.endsWith(`:${normModel}`)) {
|
|
116
|
+
winnerKey = key
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (!winnerKey && typeof run.winnerIndex === 'number' && run.winnerIndex > 0) {
|
|
120
|
+
if (cand.index === run.winnerIndex || idx === run.winnerIndex - 1) {
|
|
121
|
+
winnerKey = key
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
if (winnerKey && participantKeys.length >= 2) {
|
|
128
|
+
applyRoundEloUpdate(globalRatings, winnerKey, participantKeys, kFactor)
|
|
129
|
+
applyRoundEloUpdate(catMap, winnerKey, participantKeys, kFactor)
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return { globalRatings, categoryRatings }
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Selects the best preset for a prompt category based on historical Elo ratings of its reference models.
|
|
138
|
+
*
|
|
139
|
+
* @param {string} prompt
|
|
140
|
+
* @param {Array<object>} presets
|
|
141
|
+
* @param {object} leaderboard - Result from getMoaLeaderboard()
|
|
142
|
+
* @returns {{ presetName: string, category: string, avgElo: number } | null}
|
|
143
|
+
*/
|
|
144
|
+
export function selectBestPresetByElo(prompt = '', presets = [], leaderboard = null) {
|
|
145
|
+
if (!leaderboard || !Array.isArray(leaderboard.models) || leaderboard.models.length === 0) return null
|
|
146
|
+
if (!Array.isArray(presets) || presets.length === 0) return null
|
|
147
|
+
|
|
148
|
+
const category = classifyEloCategory(prompt)
|
|
149
|
+
const modelMap = new Map()
|
|
150
|
+
for (const m of leaderboard.models) {
|
|
151
|
+
modelMap.set(m.modelKey, m)
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
let bestPreset = null
|
|
155
|
+
let bestElo = -Infinity
|
|
156
|
+
|
|
157
|
+
for (const preset of presets) {
|
|
158
|
+
if (preset.enabled === false) continue
|
|
159
|
+
const refs = Array.isArray(preset.reference_models) ? preset.reference_models : []
|
|
160
|
+
if (refs.length === 0) continue
|
|
161
|
+
|
|
162
|
+
let sumElo = 0
|
|
163
|
+
let matchedCount = 0
|
|
164
|
+
for (const ref of refs) {
|
|
165
|
+
const key = ref.provider && ref.model ? `${ref.provider}:${ref.model}` : ref.model
|
|
166
|
+
const st = modelMap.get(key)
|
|
167
|
+
if (st) {
|
|
168
|
+
const catElo = st.categoryElo?.[category] ?? st.elo ?? INITIAL_ELO
|
|
169
|
+
sumElo += catElo
|
|
170
|
+
matchedCount++
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if (matchedCount > 0) {
|
|
175
|
+
const avgElo = sumElo / matchedCount
|
|
176
|
+
if (avgElo > bestElo && avgElo > INITIAL_ELO) {
|
|
177
|
+
bestElo = avgElo
|
|
178
|
+
bestPreset = preset.name
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if (!bestPreset) return null
|
|
184
|
+
return {
|
|
185
|
+
presetName: bestPreset,
|
|
186
|
+
category,
|
|
187
|
+
avgElo: Math.round(bestElo),
|
|
188
|
+
}
|
|
189
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quota-Aware Provider Balancing (#253)
|
|
3
|
+
* Queries provider/model rate limits and quota health via dsh-key-limits and dsh-subscriptions.
|
|
4
|
+
* Dynamically substitutes exhausted or low-quota reference slots with healthy fallback models
|
|
5
|
+
* to protect the final aggregator quota and prevent 429 pipeline aborts.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { service } from './services.js'
|
|
9
|
+
import { logger } from './logger.js'
|
|
10
|
+
import { slotLabel } from './moa-prompts.js'
|
|
11
|
+
|
|
12
|
+
const DEFAULT_FALLBACK_POOL = [
|
|
13
|
+
{ provider: 'deepseek', model: 'deepseek-reasoner' },
|
|
14
|
+
{ provider: 'deepseek', model: 'deepseek-chat' },
|
|
15
|
+
{ provider: 'qwen', model: 'qwen-2.5-coder-32b-instruct' },
|
|
16
|
+
{ provider: 'openrouter', model: 'deepseek/deepseek-r1' },
|
|
17
|
+
{ provider: 'ollama', model: 'qwen2.5-coder:32b' },
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Inspects quota status for a given { provider, model } slot via dsh-key-limits or dsh-subscriptions.
|
|
22
|
+
*
|
|
23
|
+
* @param {object} ctx - Cordis context
|
|
24
|
+
* @param {{ provider: string, model: string }} slot
|
|
25
|
+
* @param {object} [options]
|
|
26
|
+
* @returns {Promise<{ exhausted: boolean, remainingRatio: number, reason: string | null }>}
|
|
27
|
+
*/
|
|
28
|
+
export async function checkSlotQuotaStatus(ctx, slot, options = {}) {
|
|
29
|
+
const provider = String(slot?.provider || '').trim()
|
|
30
|
+
const model = String(slot?.model || '').trim()
|
|
31
|
+
if (!provider) return { exhausted: false, remainingRatio: 1, reason: null }
|
|
32
|
+
|
|
33
|
+
const minRemainingRatio = Number.isFinite(options.minRemainingRatio) ? options.minRemainingRatio : 0.1
|
|
34
|
+
|
|
35
|
+
const keyLimits = options.keyLimitsService || service(ctx, 'keyLimits')
|
|
36
|
+
if (keyLimits) {
|
|
37
|
+
try {
|
|
38
|
+
let status = null
|
|
39
|
+
if (typeof keyLimits.getQuotaStatus === 'function') {
|
|
40
|
+
status = await keyLimits.getQuotaStatus(provider, model)
|
|
41
|
+
} else if (typeof keyLimits.checkLimit === 'function') {
|
|
42
|
+
status = await keyLimits.checkLimit(provider, model)
|
|
43
|
+
} else if (typeof keyLimits.getProviderStatus === 'function') {
|
|
44
|
+
status = await keyLimits.getProviderStatus(provider)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (status) {
|
|
48
|
+
if (status.exhausted === true || status.rateLimited === true || status.blocked === true) {
|
|
49
|
+
return {
|
|
50
|
+
exhausted: true,
|
|
51
|
+
remainingRatio: 0,
|
|
52
|
+
reason: status.reason || 'Quota exhausted or rate-limited in dsh-key-limits',
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const rem = Number(status.remaining ?? status.remainingRequests ?? status.remainingTokens)
|
|
56
|
+
const lim = Number(status.limit ?? status.maxRequests ?? status.maxTokens)
|
|
57
|
+
if (Number.isFinite(rem) && Number.isFinite(lim) && lim > 0) {
|
|
58
|
+
const ratio = rem / lim
|
|
59
|
+
if (ratio < minRemainingRatio) {
|
|
60
|
+
return {
|
|
61
|
+
exhausted: true,
|
|
62
|
+
remainingRatio: ratio,
|
|
63
|
+
reason: `Low quota remaining (${Math.round(ratio * 100)}%) in dsh-key-limits`,
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return { exhausted: false, remainingRatio: ratio, reason: null }
|
|
67
|
+
}
|
|
68
|
+
if (Number.isFinite(status.remainingRatio)) {
|
|
69
|
+
if (status.remainingRatio < minRemainingRatio) {
|
|
70
|
+
return {
|
|
71
|
+
exhausted: true,
|
|
72
|
+
remainingRatio: status.remainingRatio,
|
|
73
|
+
reason: `Low quota ratio (${Math.round(status.remainingRatio * 100)}%) in dsh-key-limits`,
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return { exhausted: false, remainingRatio: status.remainingRatio, reason: null }
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
} catch (err) {
|
|
80
|
+
logger.warn(`[dsh-moa:quota] Error querying dsh-key-limits for ${provider}:${model}:`, err?.message || err)
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const subscriptions = options.subscriptionsService || service(ctx, 'subscriptions')
|
|
85
|
+
if (subscriptions) {
|
|
86
|
+
try {
|
|
87
|
+
let subStatus = null
|
|
88
|
+
if (typeof subscriptions.getSubscriptionStatus === 'function') {
|
|
89
|
+
subStatus = await subscriptions.getSubscriptionStatus(provider, model)
|
|
90
|
+
} else if (typeof subscriptions.checkQuota === 'function') {
|
|
91
|
+
subStatus = await subscriptions.checkQuota(provider, model)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (subStatus) {
|
|
95
|
+
if (subStatus.exhausted === true || subStatus.active === false || subStatus.expired === true) {
|
|
96
|
+
return {
|
|
97
|
+
exhausted: true,
|
|
98
|
+
remainingRatio: 0,
|
|
99
|
+
reason: subStatus.reason || 'Subscription quota exhausted or inactive in dsh-subscriptions',
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (Number.isFinite(subStatus.remainingRatio) && subStatus.remainingRatio < minRemainingRatio) {
|
|
103
|
+
return {
|
|
104
|
+
exhausted: true,
|
|
105
|
+
remainingRatio: subStatus.remainingRatio,
|
|
106
|
+
reason: `Low subscription quota (${Math.round(subStatus.remainingRatio * 100)}%) in dsh-subscriptions`,
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
} catch (err) {
|
|
111
|
+
logger.warn(`[dsh-moa:quota] Error querying dsh-subscriptions for ${provider}:${model}:`, err?.message || err)
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return { exhausted: false, remainingRatio: 1, reason: null }
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Balances candidate reference slots before Layer 1 execution.
|
|
120
|
+
* Preserves quota for the aggregator by substituting low-quota/exhausted slots
|
|
121
|
+
* with healthy fallback models.
|
|
122
|
+
*
|
|
123
|
+
* @param {Array<{ provider: string, model: string }>} references
|
|
124
|
+
* @param {object} options
|
|
125
|
+
* @returns {Promise<{ balancedReferences: Array<{ provider: string, model: string }>, substitutions: Array<object> }>}
|
|
126
|
+
*/
|
|
127
|
+
export async function balanceCandidateQuotas(references = [], options = {}) {
|
|
128
|
+
if (!Array.isArray(references) || references.length === 0) {
|
|
129
|
+
return { balancedReferences: [], substitutions: [] }
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const ctx = options.ctx || null
|
|
133
|
+
const aggregator = options.aggregator || null
|
|
134
|
+
const customFallbacks = Array.isArray(options.fallbackPool) && options.fallbackPool.length > 0
|
|
135
|
+
? options.fallbackPool
|
|
136
|
+
: DEFAULT_FALLBACK_POOL
|
|
137
|
+
|
|
138
|
+
const usedKeys = new Set(references.map((r) => slotLabel(r)))
|
|
139
|
+
const balancedReferences = []
|
|
140
|
+
const substitutions = []
|
|
141
|
+
|
|
142
|
+
for (let i = 0; i < references.length; i++) {
|
|
143
|
+
const slot = references[i]
|
|
144
|
+
const label = slotLabel(slot)
|
|
145
|
+
|
|
146
|
+
// Reserve higher threshold (25%) if this candidate uses the exact same provider:model as the final aggregator
|
|
147
|
+
const isSameAsAggregator = aggregator && slotLabel(aggregator) === label
|
|
148
|
+
const minRemainingRatio = isSameAsAggregator ? 0.25 : (options.minRemainingRatio ?? 0.1)
|
|
149
|
+
|
|
150
|
+
const status = await checkSlotQuotaStatus(ctx, slot, {
|
|
151
|
+
...options,
|
|
152
|
+
minRemainingRatio,
|
|
153
|
+
})
|
|
154
|
+
|
|
155
|
+
if (!status.exhausted) {
|
|
156
|
+
balancedReferences.push(slot)
|
|
157
|
+
continue
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Find a healthy replacement from customFallbacks
|
|
161
|
+
let replacement = null
|
|
162
|
+
for (const candidateFallback of customFallbacks) {
|
|
163
|
+
const fbLabel = slotLabel(candidateFallback)
|
|
164
|
+
if (fbLabel === label) continue
|
|
165
|
+
const fbStatus = await checkSlotQuotaStatus(ctx, candidateFallback, options)
|
|
166
|
+
if (!fbStatus.exhausted) {
|
|
167
|
+
replacement = { ...slot, provider: candidateFallback.provider, model: candidateFallback.model, replacedFrom: label }
|
|
168
|
+
usedKeys.add(fbLabel)
|
|
169
|
+
break
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (replacement) {
|
|
174
|
+
balancedReferences.push(replacement)
|
|
175
|
+
substitutions.push({
|
|
176
|
+
index: i + 1,
|
|
177
|
+
original: slot,
|
|
178
|
+
replacement: { provider: replacement.provider, model: replacement.model },
|
|
179
|
+
reason: status.reason,
|
|
180
|
+
})
|
|
181
|
+
logger.info(`[dsh-moa:quota] Replaced slot #${i + 1} (${label}) -> ${slotLabel(replacement)} (${status.reason})`)
|
|
182
|
+
} else {
|
|
183
|
+
balancedReferences.push(slot)
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
return { balancedReferences, substitutions }
|
|
188
|
+
}
|
package/lib/moa-router.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { selectBestPresetByElo } from './moa-elo.js'
|
|
1
2
|
/**
|
|
2
3
|
* Smart Preset Router for Mixture of Agents (MoA).
|
|
3
4
|
* Supports heuristic keyword classification and optional LLM zero-shot routing.
|
|
@@ -32,15 +33,18 @@ export function classifyPromptIntent(prompt = '') {
|
|
|
32
33
|
/**
|
|
33
34
|
* Resolves the optimal preset for a given user prompt.
|
|
34
35
|
*/
|
|
35
|
-
export async function resolvePresetForPrompt({
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
36
|
+
export async function resolvePresetForPrompt(arg1 = {}, arg2, arg3, arg4 = {}) {
|
|
37
|
+
const isPositional = typeof arg1 === "string"
|
|
38
|
+
const prompt = isPositional ? arg1 : (arg1.prompt || "")
|
|
39
|
+
const presets = isPositional ? (arg2 || []) : (arg1.presets || [])
|
|
40
|
+
const defaultPreset = isPositional ? (arg3 || "default") : (arg1.defaultPreset || "default")
|
|
41
|
+
const opts = isPositional ? (arg4 || {}) : arg1
|
|
42
|
+
const routingModel = opts.routingModel || null
|
|
43
|
+
const callLlm = opts.callLlm || null
|
|
44
|
+
const enabled = Boolean(opts.enabled ?? opts.smartRoutingEnabled ?? false)
|
|
45
|
+
const timeoutMs = opts.timeoutMs ?? 2000
|
|
46
|
+
const leaderboard = opts.leaderboard || null
|
|
47
|
+
const eloRankingEnabled = opts.eloRankingEnabled !== false
|
|
44
48
|
const availableNames = (presets || []).map((p) => p.name).filter(Boolean)
|
|
45
49
|
const safeDefault = availableNames.includes(defaultPreset) ? defaultPreset : (availableNames[0] || 'default')
|
|
46
50
|
|
|
@@ -87,6 +91,20 @@ Respond with ONLY the chosen preset name and nothing else.`
|
|
|
87
91
|
}
|
|
88
92
|
}
|
|
89
93
|
|
|
94
|
+
// 1.5. Active Learning Elo Leaderboard routing (#249)
|
|
95
|
+
if (leaderboard && eloRankingEnabled) {
|
|
96
|
+
const eloChoice = selectBestPresetByElo(prompt, presets, leaderboard)
|
|
97
|
+
if (eloChoice && availableNames.includes(eloChoice.presetName)) {
|
|
98
|
+
return {
|
|
99
|
+
presetName: eloChoice.presetName,
|
|
100
|
+
reason: "elo_active_learning",
|
|
101
|
+
category: eloChoice.category,
|
|
102
|
+
avgElo: eloChoice.avgElo,
|
|
103
|
+
isAutoRouted: true,
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
90
108
|
// 2. Keyword heuristic classifier
|
|
91
109
|
const heuristic = classifyPromptIntent(prompt)
|
|
92
110
|
if (heuristic && availableNames.includes(heuristic)) {
|
package/lib/moa-schema.js
CHANGED
|
@@ -66,6 +66,12 @@ export const PresetSchema = z.object({
|
|
|
66
66
|
uncertainty_scoring_enabled: z.boolean().default(false),
|
|
67
67
|
subagent_swarm_enabled: z.boolean().default(false),
|
|
68
68
|
adaptive_personas_enabled: z.boolean().default(false),
|
|
69
|
+
quota_aware_balancing_enabled: z.boolean().default(false),
|
|
70
|
+
self_healing_enabled: z.boolean().default(false),
|
|
71
|
+
self_healing_max_retries: z.number().default(2),
|
|
72
|
+
self_healing_command: z.string().default("npm test"),
|
|
73
|
+
visual_moa_enabled: z.boolean().default(false),
|
|
74
|
+
elo_ranking_enabled: z.boolean().default(true),
|
|
69
75
|
})
|
|
70
76
|
|
|
71
77
|
export const Config = z.object({
|
|
@@ -78,6 +84,12 @@ export const Config = z.object({
|
|
|
78
84
|
uncertainty_scoring_enabled: z.boolean().default(false).volatile(),
|
|
79
85
|
subagent_swarm_enabled: z.boolean().default(false).volatile(),
|
|
80
86
|
adaptive_personas_enabled: z.boolean().default(false).volatile(),
|
|
87
|
+
quota_aware_balancing_enabled: z.boolean().default(false).volatile(),
|
|
88
|
+
self_healing_enabled: z.boolean().default(false).volatile(),
|
|
89
|
+
self_healing_max_retries: z.number().default(2).volatile(),
|
|
90
|
+
self_healing_command: z.string().default("npm test").volatile(),
|
|
91
|
+
visual_moa_enabled: z.boolean().default(false).volatile(),
|
|
92
|
+
elo_ranking_enabled: z.boolean().default(true).volatile(),
|
|
81
93
|
smart_routing_model: ModelSlotSchema.default({ provider: '', model: '' }).volatile(),
|
|
82
94
|
prices: z.dict(PriceRow).default({}).volatile(),
|
|
83
95
|
presets: z.array(PresetSchema).default([
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Self-Healing Post-Promotion Build Loop with dsh-time-machine Auto-Rollback (#252)
|
|
3
|
+
* Executes verification command after file promotion, dispatches compiler/test error logs
|
|
4
|
+
* to the MoA aggregator for hotfix generation up to maxRetries times, and automatically
|
|
5
|
+
* rolls back to the pre-promotion dsh-time-machine snapshot if verification still fails.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { execFile } from 'node:child_process'
|
|
9
|
+
import { promisify } from 'node:util'
|
|
10
|
+
import fs from 'node:fs/promises'
|
|
11
|
+
import path from 'node:path'
|
|
12
|
+
import { logger } from './logger.js'
|
|
13
|
+
import { bestEffort } from './best-effort.js'
|
|
14
|
+
import { extractFileBlocks } from './workspace/fs.js'
|
|
15
|
+
import { isSafeRelativePath, assertPathContained, isLoopbackUrl } from './workspace/security.js'
|
|
16
|
+
|
|
17
|
+
const execFileAsync = promisify(execFile)
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Runs a verification command in the target workspace.
|
|
21
|
+
*
|
|
22
|
+
* @param {string} cwd
|
|
23
|
+
* @param {string} commandStr
|
|
24
|
+
* @param {object} [options]
|
|
25
|
+
* @returns {Promise<{ passed: boolean, exitCode: number, output: string }>}
|
|
26
|
+
*/
|
|
27
|
+
export async function runVerificationCommand(cwd, commandStr = 'npm test', options = {}) {
|
|
28
|
+
if (typeof options.execFn === 'function') {
|
|
29
|
+
return await options.execFn(cwd, commandStr)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const trimmed = String(commandStr || 'npm test').trim()
|
|
33
|
+
const isWin = process.platform === 'win32'
|
|
34
|
+
const shell = isWin ? 'cmd.exe' : '/bin/sh'
|
|
35
|
+
const shellArgs = isWin ? ['/d', '/s', '/c', trimmed] : ['-c', trimmed]
|
|
36
|
+
|
|
37
|
+
try {
|
|
38
|
+
const { stdout, stderr } = await execFileAsync(shell, shellArgs, {
|
|
39
|
+
cwd,
|
|
40
|
+
timeout: (options.timeoutSec ?? 60) * 1000,
|
|
41
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
42
|
+
signal: options.signal,
|
|
43
|
+
})
|
|
44
|
+
return {
|
|
45
|
+
passed: true,
|
|
46
|
+
exitCode: 0,
|
|
47
|
+
output: `${stdout || ''}\n${stderr || ''}`.trim(),
|
|
48
|
+
}
|
|
49
|
+
} catch (err) {
|
|
50
|
+
const out = `${err?.stdout || ''}\n${err?.stderr || ''}\n${err?.message || ''}`.trim()
|
|
51
|
+
return {
|
|
52
|
+
passed: false,
|
|
53
|
+
exitCode: typeof err?.code === 'number' ? err.code : 1,
|
|
54
|
+
output: out.slice(0, 12000),
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Triggers an automatic rollback to a pre-promotion dsh-time-machine snapshot.
|
|
61
|
+
*
|
|
62
|
+
* @param {string} baseDir
|
|
63
|
+
* @param {object} snapshot
|
|
64
|
+
* @param {object} [options]
|
|
65
|
+
* @returns {Promise<{ rolledBack: boolean, snapshotId: string | null }>}
|
|
66
|
+
*/
|
|
67
|
+
export async function rollbackToTimeMachineSnapshot(baseDir, snapshot, options = {}) {
|
|
68
|
+
const snapshotId = typeof snapshot === 'string'
|
|
69
|
+
? snapshot
|
|
70
|
+
: (snapshot?.id || snapshot?.snapshotId || snapshot?.commit || snapshot?.hash || null)
|
|
71
|
+
|
|
72
|
+
if (typeof options.rollbackFn === 'function') {
|
|
73
|
+
const res = await bestEffort('custom rollbackFn', () => options.rollbackFn(baseDir, snapshotId, snapshot))
|
|
74
|
+
return { rolledBack: Boolean(res), snapshotId }
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (options.timeMachineEngine) {
|
|
78
|
+
const fn = options.timeMachineEngine.restoreSnapshot || options.timeMachineEngine.rollback
|
|
79
|
+
if (typeof fn === 'function') {
|
|
80
|
+
const res = await bestEffort('dsh-time-machine direct restoreSnapshot', () =>
|
|
81
|
+
fn.call(options.timeMachineEngine, snapshotId, { cwd: baseDir })
|
|
82
|
+
)
|
|
83
|
+
if (res) return { rolledBack: true, snapshotId }
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const port = options.port || options.webServerPort || process.env.DSH_PORT || process.env.PORT
|
|
88
|
+
if (!port && !options.timeMachineRollbackUrl) {
|
|
89
|
+
logger.warn('[dsh-moa:self-healing] Cannot rollback: webServerPort is unknown and no timeMachineEngine/rollbackFn provided')
|
|
90
|
+
return { rolledBack: false, snapshotId }
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const endpoint = options.timeMachineRollbackUrl || `http://127.0.0.1:${port}/dsh-time-machine/restore`
|
|
94
|
+
if (!isLoopbackUrl(endpoint)) {
|
|
95
|
+
logger.warn(`[dsh-moa:self-healing] Blocked non-loopback rollback URL: ${endpoint}`)
|
|
96
|
+
return { rolledBack: false, snapshotId }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const restored = await bestEffort('dsh-time-machine HTTP rollback', async () => {
|
|
100
|
+
const res = await fetch(endpoint, {
|
|
101
|
+
method: 'POST',
|
|
102
|
+
headers: { 'Content-Type': 'application/json' },
|
|
103
|
+
body: JSON.stringify({ id: snapshotId, snapshotId, cwd: baseDir }),
|
|
104
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? 5000),
|
|
105
|
+
redirect: 'error',
|
|
106
|
+
})
|
|
107
|
+
return res.ok
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
return { rolledBack: Boolean(restored), snapshotId }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Applies extracted hotfix files directly to the workspace directory.
|
|
115
|
+
*/
|
|
116
|
+
async function applyHotfixFiles(cwd, files = []) {
|
|
117
|
+
const written = []
|
|
118
|
+
for (const f of files) {
|
|
119
|
+
const rel = f.relativePath || f.path
|
|
120
|
+
if (!rel || !isSafeRelativePath(rel)) continue
|
|
121
|
+
const targetPath = assertPathContained(cwd, rel)
|
|
122
|
+
await fs.mkdir(path.dirname(targetPath), { recursive: true })
|
|
123
|
+
await fs.writeFile(targetPath, f.content, 'utf8')
|
|
124
|
+
written.push(rel)
|
|
125
|
+
}
|
|
126
|
+
return written
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Runs the post-promotion self-healing build loop.
|
|
131
|
+
*
|
|
132
|
+
* @param {object} params
|
|
133
|
+
* @returns {Promise<{
|
|
134
|
+
* enabled: boolean,
|
|
135
|
+
* passed: boolean,
|
|
136
|
+
* attempts: number,
|
|
137
|
+
* rolledBack: boolean,
|
|
138
|
+
* snapshotId: string | null,
|
|
139
|
+
* hotfixFiles: string[],
|
|
140
|
+
* lastOutput: string
|
|
141
|
+
* }>}
|
|
142
|
+
*/
|
|
143
|
+
export async function runSelfHealingLoop({
|
|
144
|
+
cwd,
|
|
145
|
+
command = 'npm test',
|
|
146
|
+
maxRetries = 2,
|
|
147
|
+
aggregatorSlot,
|
|
148
|
+
callLlm,
|
|
149
|
+
userPrompt = '',
|
|
150
|
+
promotedFiles = [],
|
|
151
|
+
checkpoint = null,
|
|
152
|
+
promoteOpts = {},
|
|
153
|
+
onProgress = null,
|
|
154
|
+
signal = null,
|
|
155
|
+
execFn = null,
|
|
156
|
+
rollbackFn = null,
|
|
157
|
+
}) {
|
|
158
|
+
const retries = Math.max(0, Math.min(5, Number(maxRetries ?? 2)))
|
|
159
|
+
const hotfixFiles = new Set()
|
|
160
|
+
|
|
161
|
+
if (typeof onProgress === 'function') {
|
|
162
|
+
onProgress(`🩺 *Self-Healing Verification: Running \`${command}\`...*\n`)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
let check = await runVerificationCommand(cwd, command, { execFn, signal })
|
|
166
|
+
if (check.passed) {
|
|
167
|
+
return {
|
|
168
|
+
enabled: true,
|
|
169
|
+
passed: true,
|
|
170
|
+
attempts: 0,
|
|
171
|
+
rolledBack: false,
|
|
172
|
+
snapshotId: null,
|
|
173
|
+
hotfixFiles: [],
|
|
174
|
+
lastOutput: check.output,
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
let attempts = 0
|
|
179
|
+
while (!check.passed && attempts < retries) {
|
|
180
|
+
if (signal?.aborted) break
|
|
181
|
+
attempts++
|
|
182
|
+
|
|
183
|
+
if (typeof onProgress === 'function') {
|
|
184
|
+
onProgress(`🔧 *Self-Healing Attempt ${attempts}/${retries}: Requesting hotfix patch from aggregator...*\n`)
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const fileList = Array.isArray(promotedFiles)
|
|
188
|
+
? promotedFiles.map((f) => (typeof f === 'string' ? f : f.relativePath || f.path)).filter(Boolean)
|
|
189
|
+
: []
|
|
190
|
+
|
|
191
|
+
const hotfixPrompt = [
|
|
192
|
+
`The promoted MoA changes failed the verification command \`${command}\` (exit code ${check.exitCode}).`,
|
|
193
|
+
`Original User Task: ${userPrompt}`,
|
|
194
|
+
`Promoted Files: ${fileList.join(', ') || 'workspace files'}`,
|
|
195
|
+
`\n### Compiler / Test Failure Output:\n\`\`\`\n${check.output}\n\`\`\``,
|
|
196
|
+
`\nProvide a minimal hotfix patch using markdown code blocks with \`path=<relative/path>\` header so the build and tests pass.`,
|
|
197
|
+
].join('\n')
|
|
198
|
+
|
|
199
|
+
const llmRes = await callLlm({
|
|
200
|
+
provider: aggregatorSlot?.provider || 'default',
|
|
201
|
+
model: aggregatorSlot?.model || 'default',
|
|
202
|
+
messages: [
|
|
203
|
+
{
|
|
204
|
+
role: 'system',
|
|
205
|
+
content: 'You are an expert automated build healer. Fix compilation or unit test errors by outputting complete corrected files with ```lang path=relative/path.ext fences.',
|
|
206
|
+
},
|
|
207
|
+
{ role: 'user', content: hotfixPrompt },
|
|
208
|
+
],
|
|
209
|
+
temperature: 0.2,
|
|
210
|
+
maxTokens: 4096,
|
|
211
|
+
signal,
|
|
212
|
+
})
|
|
213
|
+
|
|
214
|
+
const patchText = typeof llmRes === 'string' ? llmRes : (llmRes?.content || llmRes?.text || '')
|
|
215
|
+
const extracted = extractFileBlocks(patchText)
|
|
216
|
+
if (extracted.length > 0) {
|
|
217
|
+
const applied = await applyHotfixFiles(cwd, extracted)
|
|
218
|
+
for (const f of applied) hotfixFiles.add(f)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
check = await runVerificationCommand(cwd, command, { execFn, signal })
|
|
222
|
+
if (check.passed) {
|
|
223
|
+
if (typeof onProgress === 'function') {
|
|
224
|
+
onProgress(`✅ *Self-Healing Succeeded on attempt ${attempts}/${retries}!*\n`)
|
|
225
|
+
}
|
|
226
|
+
return {
|
|
227
|
+
enabled: true,
|
|
228
|
+
passed: true,
|
|
229
|
+
attempts,
|
|
230
|
+
rolledBack: false,
|
|
231
|
+
snapshotId: null,
|
|
232
|
+
hotfixFiles: [...hotfixFiles],
|
|
233
|
+
lastOutput: check.output,
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// Verification still failed after maxRetries -> unconditional Time Machine rollback
|
|
239
|
+
const snapshotObj = checkpoint || promoteOpts?.checkpointSnapshot || (Array.isArray(promotedFiles) && promotedFiles._checkpoint) || null
|
|
240
|
+
if (typeof onProgress === 'function') {
|
|
241
|
+
onProgress(`⏪ *Self-Healing exhausted (${attempts}/${retries}). Rolling back workspace via dsh-time-machine...*\n`)
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const rb = await rollbackToTimeMachineSnapshot(cwd, snapshotObj, {
|
|
245
|
+
...promoteOpts,
|
|
246
|
+
rollbackFn,
|
|
247
|
+
})
|
|
248
|
+
|
|
249
|
+
return {
|
|
250
|
+
enabled: true,
|
|
251
|
+
passed: false,
|
|
252
|
+
attempts,
|
|
253
|
+
rolledBack: rb.rolledBack,
|
|
254
|
+
snapshotId: rb.snapshotId,
|
|
255
|
+
hotfixFiles: [...hotfixFiles],
|
|
256
|
+
lastOutput: check.output,
|
|
257
|
+
}
|
|
258
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multimodal Visual MoA Consensus via dsh-vision-bridge and dsh-live-canvas (#250)
|
|
3
|
+
* Renders UI/Frontend candidates (HTML, React, CSS, SVG) via dsh-live-canvas,
|
|
4
|
+
* captures headless screenshots, and evaluates layout/responsiveness via dsh-vision-bridge.
|
|
5
|
+
* Gracefully degrades to text evaluation when companion plugins are unavailable.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { service } from './services.js'
|
|
9
|
+
import { logger } from './logger.js'
|
|
10
|
+
import { slotLabel } from './moa-prompts.js'
|
|
11
|
+
|
|
12
|
+
const UI_TASK_REGEX = /\b(html|css|svg|ui|frontend|react|vue|tailwind|layout|landing|page|component|button|modal|responsive|card|dashboard|widget)\b/i
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Checks whether a prompt or candidate output involves UI/Frontend rendering.
|
|
16
|
+
*/
|
|
17
|
+
export function isVisualUiTask(prompt = '', referenceOutputs = []) {
|
|
18
|
+
if (UI_TASK_REGEX.test(String(prompt || ''))) return true
|
|
19
|
+
for (const ref of referenceOutputs || []) {
|
|
20
|
+
const files = Array.isArray(ref?.files) ? ref.files : []
|
|
21
|
+
if (files.some((f) => /\.(html?|css|svg|jsx|tsx|vue)$/i.test(f.relativePath || f.path || ''))) {
|
|
22
|
+
return true
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return false
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Extracts renderable HTML/SVG preview markup from a candidate output.
|
|
30
|
+
*/
|
|
31
|
+
export function extractRenderableMarkup(candidate = {}) {
|
|
32
|
+
const files = Array.isArray(candidate.files) ? candidate.files : []
|
|
33
|
+
const htmlFile = files.find((f) => /\.(html?|svg)$/i.test(f.relativePath || f.path || ''))
|
|
34
|
+
if (htmlFile?.content) {
|
|
35
|
+
return { title: htmlFile.relativePath || 'index.html', content: htmlFile.content }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const cssFile = files.find((f) => /\.css$/i.test(f.relativePath || f.path || ''))
|
|
39
|
+
const jsFile = files.find((f) => /\.(jsx?|tsx?)$/i.test(f.relativePath || f.path || ''))
|
|
40
|
+
if (cssFile || jsFile) {
|
|
41
|
+
const html = [
|
|
42
|
+
'<!DOCTYPE html><html><head><meta charset="utf-8">',
|
|
43
|
+
cssFile ? `<style>${cssFile.content}</style>` : '',
|
|
44
|
+
'</head><body><div id="app">UI Preview</div>',
|
|
45
|
+
'</body></html>',
|
|
46
|
+
].join('\n')
|
|
47
|
+
return { title: 'synthesized-preview.html', content: html }
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const text = String(candidate.text || '')
|
|
51
|
+
const fenceMatch = /```(?:html|svg)\s*([\s\S]*?)```/i.exec(text)
|
|
52
|
+
if (fenceMatch) {
|
|
53
|
+
return { title: 'inline-preview.html', content: fenceMatch[1].trim() }
|
|
54
|
+
}
|
|
55
|
+
return null
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Evaluates candidates visually using dsh-live-canvas and dsh-vision-bridge.
|
|
60
|
+
* Returns { degraded: true } if companion services are absent so the caller falls back cleanly.
|
|
61
|
+
*
|
|
62
|
+
* @param {object} params
|
|
63
|
+
* @returns {Promise<{
|
|
64
|
+
* degraded: boolean,
|
|
65
|
+
* reason?: string,
|
|
66
|
+
* visualScores: Array<{ candidateIndex: number, label: string, visualScore: number, critique: string, previewUrl: string | null }>,
|
|
67
|
+
* directive: string
|
|
68
|
+
* }>}
|
|
69
|
+
*/
|
|
70
|
+
export async function runVisualMoaConsensus({
|
|
71
|
+
ctx = null,
|
|
72
|
+
liveCanvas = null,
|
|
73
|
+
visionBridge = null,
|
|
74
|
+
userPrompt = '',
|
|
75
|
+
referenceOutputs = [],
|
|
76
|
+
onProgress = null,
|
|
77
|
+
}) {
|
|
78
|
+
const resolvedCanvas = liveCanvas || service(ctx, 'liveCanvas')
|
|
79
|
+
const resolvedVision = visionBridge || service(ctx, 'visionBridge')
|
|
80
|
+
|
|
81
|
+
if (!resolvedCanvas && !resolvedVision) {
|
|
82
|
+
return {
|
|
83
|
+
degraded: true,
|
|
84
|
+
reason: 'Companion plugins dsh-live-canvas and dsh-vision-bridge are not active; using standard text evaluation.',
|
|
85
|
+
visualScores: [],
|
|
86
|
+
directive: '',
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (typeof onProgress === 'function') {
|
|
91
|
+
onProgress('👁️ *Visual MoA Consensus: Rendering UI candidates in Live Canvas & evaluating via Vision Bridge...*\n')
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const visualScores = []
|
|
95
|
+
for (let i = 0; i < referenceOutputs.length; i++) {
|
|
96
|
+
const ref = referenceOutputs[i]
|
|
97
|
+
if (!ref || ref.ok === false) continue
|
|
98
|
+
const idx = ref.index ?? (i + 1)
|
|
99
|
+
const label = ref.label || slotLabel(ref.slot || ref)
|
|
100
|
+
const markup = extractRenderableMarkup(ref)
|
|
101
|
+
if (!markup) continue
|
|
102
|
+
|
|
103
|
+
let previewUrl = null
|
|
104
|
+
let screenshot = null
|
|
105
|
+
|
|
106
|
+
if (resolvedCanvas) {
|
|
107
|
+
try {
|
|
108
|
+
if (typeof resolvedCanvas.createPreviewFromContent === 'function') {
|
|
109
|
+
const prev = await resolvedCanvas.createPreviewFromContent({
|
|
110
|
+
content: markup.content,
|
|
111
|
+
title: `Candidate #${idx} (${label}) - ${markup.title}`,
|
|
112
|
+
})
|
|
113
|
+
previewUrl = prev?.previewUrl || null
|
|
114
|
+
screenshot = prev?.screenshot || prev?.imageBase64 || null
|
|
115
|
+
}
|
|
116
|
+
if (!screenshot && typeof resolvedCanvas.captureScreenshot === 'function') {
|
|
117
|
+
screenshot = await resolvedCanvas.captureScreenshot({
|
|
118
|
+
content: markup.content,
|
|
119
|
+
previewUrl,
|
|
120
|
+
})
|
|
121
|
+
}
|
|
122
|
+
} catch (err) {
|
|
123
|
+
logger.warn(`[dsh-moa:visual] Live Canvas preview failed for candidate #${idx}:`, err?.message || err)
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
let visualScore = 7.5
|
|
128
|
+
let critique = 'Rendered cleanly in Live Canvas preview.'
|
|
129
|
+
|
|
130
|
+
if (resolvedVision) {
|
|
131
|
+
try {
|
|
132
|
+
const evalFn = resolvedVision.evaluateVisualLayout || resolvedVision.analyzeImage || resolvedVision.inspectUi
|
|
133
|
+
if (typeof evalFn === 'function') {
|
|
134
|
+
const vRes = await evalFn.call(resolvedVision, {
|
|
135
|
+
prompt: userPrompt,
|
|
136
|
+
candidateIndex: idx,
|
|
137
|
+
html: markup.content,
|
|
138
|
+
previewUrl,
|
|
139
|
+
screenshot,
|
|
140
|
+
})
|
|
141
|
+
if (vRes) {
|
|
142
|
+
if (Number.isFinite(Number(vRes.score))) {
|
|
143
|
+
visualScore = Math.max(1, Math.min(10, Number(vRes.score)))
|
|
144
|
+
}
|
|
145
|
+
if (vRes.critique || vRes.summary || vRes.feedback) {
|
|
146
|
+
critique = String(vRes.critique || vRes.summary || vRes.feedback).trim()
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
} catch (err) {
|
|
151
|
+
logger.warn(`[dsh-moa:visual] Vision Bridge evaluation failed for candidate #${idx}:`, err?.message || err)
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
visualScores.push({
|
|
156
|
+
candidateIndex: idx,
|
|
157
|
+
label,
|
|
158
|
+
visualScore: Number(visualScore.toFixed(1)),
|
|
159
|
+
critique,
|
|
160
|
+
previewUrl,
|
|
161
|
+
})
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (visualScores.length === 0) {
|
|
165
|
+
return {
|
|
166
|
+
degraded: true,
|
|
167
|
+
reason: 'No renderable HTML/CSS/SVG artifacts found in candidates.',
|
|
168
|
+
visualScores: [],
|
|
169
|
+
directive: '',
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const lines = [
|
|
174
|
+
'### 👁️ Multimodal Visual MoA Consensus (`dsh-live-canvas` + `dsh-vision-bridge`):',
|
|
175
|
+
'The following UI candidates were rendered and visually inspected for layout, responsiveness, and design fidelity:',
|
|
176
|
+
...visualScores.map(
|
|
177
|
+
(v) => `- **Candidate #${v.candidateIndex} (${v.label})**: Visual Score **${v.visualScore}/10** — ${v.critique}${v.previewUrl ? ` ([Live Preview](${v.previewUrl}))` : ''}`
|
|
178
|
+
),
|
|
179
|
+
'Prioritize candidates with higher visual layout fidelity and zero rendering defects.',
|
|
180
|
+
]
|
|
181
|
+
|
|
182
|
+
return {
|
|
183
|
+
degraded: false,
|
|
184
|
+
visualScores,
|
|
185
|
+
directive: lines.join('\n'),
|
|
186
|
+
}
|
|
187
|
+
}
|
package/lib/moa-workbench.js
CHANGED
|
@@ -27,6 +27,30 @@ export const COMPANION_PLUGINS = [
|
|
|
27
27
|
url: 'https://github.com/goodandready/dsh-time-machine',
|
|
28
28
|
description: 'Pre-promotion shadow Git checkpoints and rollback points',
|
|
29
29
|
},
|
|
30
|
+
{
|
|
31
|
+
id: 'dsh-key-limits',
|
|
32
|
+
name: 'DSH Key Limits',
|
|
33
|
+
url: 'https://github.com/GooDAnDReaDY/dsh-key-limits',
|
|
34
|
+
description: 'Provider API key rate-limit and quota telemetry',
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
id: 'dsh-subscriptions',
|
|
38
|
+
name: 'DSH Subscriptions',
|
|
39
|
+
url: 'https://github.com/GooDAnDReaDY/dsh-subscriptions',
|
|
40
|
+
description: 'Subscription tier and quota tracking for LLM providers',
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
id: 'dsh-vision-bridge',
|
|
44
|
+
name: 'DSH Vision Bridge',
|
|
45
|
+
url: 'https://github.com/GooDAnDReaDY/dsh-vision-bridge',
|
|
46
|
+
description: 'Multimodal VLM visual evaluation for UI/Frontend artifacts',
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
id: 'dsh-live-canvas',
|
|
50
|
+
name: 'DSH Live Canvas',
|
|
51
|
+
url: 'https://github.com/GooDAnDReaDY/dsh-live-canvas',
|
|
52
|
+
description: 'Isolated live HTML/CSS/SVG preview environment and screenshot capture',
|
|
53
|
+
},
|
|
30
54
|
]
|
|
31
55
|
|
|
32
56
|
/**
|
package/lib/pipeline/context.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { balanceCandidateQuotas } from '../moa-quota-balancer.js'
|
|
1
2
|
import { collectProjectContext, formatProjectContext, isRefinementTask } from '../workspace/scanner.js'
|
|
2
3
|
import { applyBudgetGuardrails, createBudgetAbortPayload } from '../moa-budget.js'
|
|
3
4
|
import { extractPriorTurnBaseline, pruneMultiTurnMessages } from '../moa-context.js'
|
|
@@ -13,6 +14,7 @@ export async function stageProjectContext({
|
|
|
13
14
|
maxTokens,
|
|
14
15
|
startTime,
|
|
15
16
|
onProgress,
|
|
17
|
+
ctx = null,
|
|
16
18
|
}) {
|
|
17
19
|
const rawRefs = (Array.isArray(preset?.reference_models) && preset.reference_models.length > 0)
|
|
18
20
|
? preset.reference_models
|
|
@@ -79,7 +81,23 @@ export async function stageProjectContext({
|
|
|
79
81
|
return { ok: false, abortPayload }
|
|
80
82
|
}
|
|
81
83
|
|
|
82
|
-
|
|
84
|
+
let effectiveReferences = budgetCheck.references
|
|
85
|
+
let quotaSubstitutions = []
|
|
86
|
+
if (preset?.quota_aware_balancing_enabled) {
|
|
87
|
+
const fallbackPool = Array.isArray(preset?.local_fallback_models) && preset.local_fallback_models.length > 0
|
|
88
|
+
? preset.local_fallback_models
|
|
89
|
+
: undefined
|
|
90
|
+
const bal = await balanceCandidateQuotas(effectiveReferences, {
|
|
91
|
+
ctx,
|
|
92
|
+
aggregator: primaryJudge,
|
|
93
|
+
fallbackPool,
|
|
94
|
+
})
|
|
95
|
+
effectiveReferences = bal.balancedReferences
|
|
96
|
+
quotaSubstitutions = bal.substitutions
|
|
97
|
+
if (quotaSubstitutions.length > 0 && typeof onProgress === "function") {
|
|
98
|
+
onProgress(`⚖️ *Quota-Aware Balancing: Replaced ${quotaSubstitutions.length} low-quota slot(s) to preserve aggregator budget.*\n`)
|
|
99
|
+
}
|
|
100
|
+
}
|
|
83
101
|
if (effectiveReferences.length === 1 && !preset?.curator_synthesis) {
|
|
84
102
|
isFastMode = true
|
|
85
103
|
}
|
|
@@ -103,7 +121,7 @@ export async function stageProjectContext({
|
|
|
103
121
|
isStreamAggregator, isQuorumEnabled, gracePeriodSec, candidateRetries,
|
|
104
122
|
refTimeoutSec, aggTimeoutSec, isBlindEvaluation, isPeerCritiqueEnabled,
|
|
105
123
|
allowCandidateOverride, collectedCtx, isRefinement, projectContext,
|
|
106
|
-
effectiveReferences, priorTurnBaseline, candidateSystemPrompt,
|
|
124
|
+
effectiveReferences, quotaSubstitutions, priorTurnBaseline, candidateSystemPrompt,
|
|
107
125
|
needsQuestions, enrichedMessages,
|
|
108
126
|
}
|
|
109
127
|
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { classifyEloCategory } from '../moa-elo.js'
|
|
2
|
+
import { runSelfHealingLoop } from '../moa-self-healing.js'
|
|
1
3
|
import { logger } from '../logger.js'
|
|
2
4
|
import { parseWinnerIndex, parseRecommendedAssembler } from '../moa-parser.js'
|
|
3
5
|
import { extractFileBlocks, writeCandidateWorkspace, promoteCandidateWorkspace, cleanMoaWorkspaces } from '../workspace/fs.js'
|
|
@@ -33,6 +35,10 @@ export async function stagePromotion({
|
|
|
33
35
|
historyFilePath,
|
|
34
36
|
collectedCtx,
|
|
35
37
|
ctx = null,
|
|
38
|
+
callLlm = null,
|
|
39
|
+
onProgress = null,
|
|
40
|
+
execFn = null,
|
|
41
|
+
visualResult = null,
|
|
36
42
|
}) {
|
|
37
43
|
const winningIndex = (multiJudgeResult?.consensus && multiJudgeResult?.winningCandidateIndex)
|
|
38
44
|
? multiJudgeResult.winningCandidateIndex
|
|
@@ -69,6 +75,24 @@ export async function stagePromotion({
|
|
|
69
75
|
throw promoteErr
|
|
70
76
|
}
|
|
71
77
|
|
|
78
|
+
let selfHealingResult = null
|
|
79
|
+
if (preset?.self_healing_enabled && promotedFiles.length > 0 && typeof callLlm === "function") {
|
|
80
|
+
selfHealingResult = await runSelfHealingLoop({
|
|
81
|
+
cwd,
|
|
82
|
+
command: preset.self_healing_command || "npm test",
|
|
83
|
+
maxRetries: preset.self_healing_max_retries ?? 2,
|
|
84
|
+
aggregatorSlot: chosenJudge,
|
|
85
|
+
callLlm,
|
|
86
|
+
userPrompt,
|
|
87
|
+
promotedFiles,
|
|
88
|
+
checkpoint: promotedFiles.checkpoint || null,
|
|
89
|
+
promoteOpts,
|
|
90
|
+
onProgress,
|
|
91
|
+
signal,
|
|
92
|
+
execFn,
|
|
93
|
+
})
|
|
94
|
+
}
|
|
95
|
+
|
|
72
96
|
const livePreview = await createPromotedPreview(liveCanvas, cwd, promotedFiles, referenceOutputs, { winningIndex, synthesizedFiles })
|
|
73
97
|
const durationMs = Date.now() - startTime
|
|
74
98
|
const usageSummary = summarizeMoAUsage(referenceOutputs, aggUsage, multiJudgeResult?.totalUsage)
|
|
@@ -100,7 +124,7 @@ export async function stagePromotion({
|
|
|
100
124
|
|
|
101
125
|
try {
|
|
102
126
|
await recordMoaRunAsync({
|
|
103
|
-
id: runId, cwd, prompt: userPrompt, preset: preset?.name || 'default', isRefinement,
|
|
127
|
+
id: runId, cwd, prompt: userPrompt, category: classifyEloCategory(userPrompt), preset: preset?.name || 'default', isRefinement,
|
|
104
128
|
candidates: candidatesForHistory(referenceOutputs),
|
|
105
129
|
aggregator: { provider: chosenJudge.provider, model: chosenJudge.model, usage: aggUsage, costUsd: aggUsage.costUsd },
|
|
106
130
|
winnerIndex: winningIndex, winnerModel, promotedFiles, benchmarkReport, consensus: multiJudgeResult,
|
|
@@ -125,7 +149,7 @@ export async function stagePromotion({
|
|
|
125
149
|
kind: 'synthesis', content: synthesizedText, aggregator: isFastMode ? 'Fast Mode (Direct)' : finalAggLabel,
|
|
126
150
|
references: referenceOutputs, presetName: preset?.name || 'default', isRefinement, isFastMode, isCuratorSynthesis,
|
|
127
151
|
recommendedAssembler, winningIndex, winnerModel, promotedFiles, runId, allowCandidateOverride,
|
|
128
|
-
benchmarkReport, consensus: multiJudgeResult, fileScores, ...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
152
|
+
benchmarkReport, consensus: multiJudgeResult, fileScores, ...(selfHealingResult ? { selfHealing: selfHealingResult } : {}), ...(visualResult ? { visualConsensus: visualResult } : {}), ...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
129
153
|
usage: usageSummary, skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs,
|
|
130
154
|
}
|
|
131
155
|
}
|
package/lib/pipeline/runner.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { runVisualMoaConsensus } from '../moa-visual-consensus.js'
|
|
1
2
|
import crypto from 'node:crypto'
|
|
2
3
|
import { isTestEnvironment } from '../history.js'
|
|
3
4
|
import { cleanMoaWorkspaces } from '../workspace/fs.js'
|
|
@@ -40,7 +41,7 @@ export async function runMoAPipeline({
|
|
|
40
41
|
|
|
41
42
|
// Stage 1: Context & Configuration
|
|
42
43
|
const ctxStage = await stageProjectContext({
|
|
43
|
-
userPrompt, messages, preset, cwd, prices, maxTokens: preset?.max_tokens, startTime, onProgress,
|
|
44
|
+
userPrompt, messages, preset, cwd, prices, maxTokens: preset?.max_tokens, startTime, onProgress, ctx,
|
|
44
45
|
})
|
|
45
46
|
if (!ctxStage.ok) return ctxStage.abortPayload
|
|
46
47
|
|
|
@@ -157,9 +158,20 @@ export async function runMoAPipeline({
|
|
|
157
158
|
const compositeMergeDirective = (preset?.composite_merge_enabled && successfulRefs.length > 1)
|
|
158
159
|
? buildCompositeBlockDirectives(successfulRefs, { blind: Boolean(preset.blind_evaluation) })
|
|
159
160
|
: null
|
|
160
|
-
|
|
161
|
+
let visualResult = null
|
|
162
|
+
if (preset?.visual_moa_enabled) {
|
|
163
|
+
visualResult = await runVisualMoaConsensus({
|
|
164
|
+
ctx,
|
|
165
|
+
liveCanvas,
|
|
166
|
+
userPrompt,
|
|
167
|
+
referenceOutputs,
|
|
168
|
+
onProgress,
|
|
169
|
+
})
|
|
170
|
+
}
|
|
171
|
+
const rawFileDirective = (preset?.interactive_workbench_enabled)
|
|
161
172
|
? formatFileScoresDirective()
|
|
162
173
|
: null
|
|
174
|
+
const fileScoringDirective = [rawFileDirective, visualResult?.directive].filter(Boolean).join("\n\n") || null
|
|
163
175
|
|
|
164
176
|
// Stage 7: Synthesis Phase
|
|
165
177
|
const synthRes = await stageSynthesis({
|
|
@@ -189,7 +201,7 @@ export async function runMoAPipeline({
|
|
|
189
201
|
promoteOpts, cwd, runId, signal, liveCanvas, startTime, chosenJudge, aggUsage,
|
|
190
202
|
userPrompt, preset, isRefinement: ctxStage.isRefinement, isFastMode: ctxStage.isFastMode,
|
|
191
203
|
effectiveReferences: ctxStage.effectiveReferences, referenceModels: ctxStage.referenceModels,
|
|
192
|
-
historyFilePath, collectedCtx: ctxStage.collectedCtx, ctx,
|
|
204
|
+
historyFilePath, collectedCtx: ctxStage.collectedCtx, ctx, callLlm, onProgress, execFn, visualResult,
|
|
193
205
|
})
|
|
194
206
|
if (!promRes.ok) {
|
|
195
207
|
if (promRes.abort) return { error: promRes.error }
|