@goodandready/dsh-moa 0.2.68 → 0.2.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +69 -0
- package/lib/moa-quota-balancer.js +188 -0
- package/lib/moa-schema.js +8 -0
- package/lib/moa-self-healing.js +258 -0
- package/lib/moa-workbench.js +12 -0
- package/lib/pipeline/context.js +20 -2
- package/lib/pipeline/promotion.js +23 -1
- package/lib/pipeline/runner.js +2 -2
- package/package.json +1 -1
package/lib/client.js
CHANGED
|
@@ -2017,6 +2017,7 @@ window.__ModuleLoader__.load({
|
|
|
2017
2017
|
t("aggregator.report_desc") || "Generate downloadable Markdown & JSON reports detailing candidate latency, agreement, test gate, and cost breakdown."
|
|
2018
2018
|
)
|
|
2019
2019
|
),
|
|
2020
|
+
// --- src/client/85-card-aggregator-ext.js ---
|
|
2020
2021
|
/* Interactive Workbench & File Scoring (#251) */
|
|
2021
2022
|
React.createElement(
|
|
2022
2023
|
"div",
|
|
@@ -2054,6 +2055,74 @@ window.__ModuleLoader__.load({
|
|
|
2054
2055
|
"(" + (t("aggregator.workbench_companion_note") || "safe shadow Git checkpoints before file promotion") + ")"
|
|
2055
2056
|
)
|
|
2056
2057
|
),
|
|
2058
|
+
/* Quota-Aware Provider Balancing (#253) */
|
|
2059
|
+
React.createElement(
|
|
2060
|
+
"div",
|
|
2061
|
+
{ style: { display: "flex", flexDirection: "column", gap: 6, padding: "8px 12px", background: "var(--dsw-alias-bg-layer-2)", borderRadius: 8, border: "1px solid var(--dsw-alias-border-l2)", marginTop: 8 } },
|
|
2062
|
+
React.createElement(
|
|
2063
|
+
"label",
|
|
2064
|
+
{ style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer", fontSize: 13, fontWeight: 500 } },
|
|
2065
|
+
React.createElement("input", {
|
|
2066
|
+
type: "checkbox",
|
|
2067
|
+
checked: Boolean(currentPreset.quota_aware_balancing_enabled),
|
|
2068
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, quota_aware_balancing_enabled: e.target.checked })),
|
|
2069
|
+
style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
|
|
2070
|
+
}),
|
|
2071
|
+
t("aggregator.quota_balancing_title") || "\u2696\uFE0F Quota-Aware Provider Balancing"
|
|
2072
|
+
),
|
|
2073
|
+
React.createElement(
|
|
2074
|
+
"div",
|
|
2075
|
+
{ style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
|
|
2076
|
+
t("aggregator.quota_balancing_desc") || "Automatically substitute low-quota candidate models in Layer 1 to preserve rate limits for the final aggregator."
|
|
2077
|
+
),
|
|
2078
|
+
React.createElement(
|
|
2079
|
+
"div",
|
|
2080
|
+
{ style: { fontSize: 11, color: "var(--dsw-alias-label-tertiary)", marginLeft: 22, marginTop: 2, display: "flex", alignItems: "center", gap: 6, flexWrap: "wrap" } },
|
|
2081
|
+
"\u{1F517} Companion Plugins:",
|
|
2082
|
+
React.createElement(
|
|
2083
|
+
"a",
|
|
2084
|
+
{ href: "https://github.com/GooDAnDReaDY/dsh-key-limits", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
|
|
2085
|
+
"@goodandready/dsh-key-limits"
|
|
2086
|
+
),
|
|
2087
|
+
"/",
|
|
2088
|
+
React.createElement(
|
|
2089
|
+
"a",
|
|
2090
|
+
{ href: "https://github.com/GooDAnDReaDY/dsh-subscriptions", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
|
|
2091
|
+
"@goodandready/dsh-subscriptions"
|
|
2092
|
+
)
|
|
2093
|
+
)
|
|
2094
|
+
),
|
|
2095
|
+
/* Self-Healing Post-Promotion Build Loop (#252) */
|
|
2096
|
+
React.createElement(
|
|
2097
|
+
"div",
|
|
2098
|
+
{ style: { display: "flex", flexDirection: "column", gap: 6, padding: "8px 12px", background: "var(--dsw-alias-bg-layer-2)", borderRadius: 8, border: "1px solid var(--dsw-alias-border-l2)", marginTop: 8 } },
|
|
2099
|
+
React.createElement(
|
|
2100
|
+
"label",
|
|
2101
|
+
{ style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer", fontSize: 13, fontWeight: 500 } },
|
|
2102
|
+
React.createElement("input", {
|
|
2103
|
+
type: "checkbox",
|
|
2104
|
+
checked: Boolean(currentPreset.self_healing_enabled),
|
|
2105
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, self_healing_enabled: e.target.checked })),
|
|
2106
|
+
style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
|
|
2107
|
+
}),
|
|
2108
|
+
t("aggregator.self_healing_title") || "\u{1FA7A} Self-Healing Post-Promotion Build Loop"
|
|
2109
|
+
),
|
|
2110
|
+
React.createElement(
|
|
2111
|
+
"div",
|
|
2112
|
+
{ style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
|
|
2113
|
+
t("aggregator.self_healing_desc") || "Verify promoted files with a test command, request hotfix patches from the aggregator on failure, or auto-rollback to the Time Machine checkpoint."
|
|
2114
|
+
),
|
|
2115
|
+
React.createElement(
|
|
2116
|
+
"div",
|
|
2117
|
+
{ style: { fontSize: 11, color: "var(--dsw-alias-label-tertiary)", marginLeft: 22, marginTop: 2, display: "flex", alignItems: "center", gap: 6 } },
|
|
2118
|
+
"\u{1F517} Companion Plugin:",
|
|
2119
|
+
React.createElement(
|
|
2120
|
+
"a",
|
|
2121
|
+
{ href: "https://github.com/GooDAnDReaDY/dsh-time-machine", target: "_blank", rel: "noopener noreferrer", style: { color: "var(--dsw-alias-state-brand-primary)", textDecoration: "underline" } },
|
|
2122
|
+
"@goodandready/dsh-time-machine"
|
|
2123
|
+
)
|
|
2124
|
+
)
|
|
2125
|
+
),
|
|
2057
2126
|
/* Graceful Degradation & Local Fallback (#175, #176) */
|
|
2058
2127
|
React.createElement(
|
|
2059
2128
|
"div",
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quota-Aware Provider Balancing (#253)
|
|
3
|
+
* Queries provider/model rate limits and quota health via dsh-key-limits and dsh-subscriptions.
|
|
4
|
+
* Dynamically substitutes exhausted or low-quota reference slots with healthy fallback models
|
|
5
|
+
* to protect the final aggregator quota and prevent 429 pipeline aborts.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { service } from './services.js'
|
|
9
|
+
import { logger } from './logger.js'
|
|
10
|
+
import { slotLabel } from './moa-prompts.js'
|
|
11
|
+
|
|
12
|
+
const DEFAULT_FALLBACK_POOL = [
|
|
13
|
+
{ provider: 'deepseek', model: 'deepseek-reasoner' },
|
|
14
|
+
{ provider: 'deepseek', model: 'deepseek-chat' },
|
|
15
|
+
{ provider: 'qwen', model: 'qwen-2.5-coder-32b-instruct' },
|
|
16
|
+
{ provider: 'openrouter', model: 'deepseek/deepseek-r1' },
|
|
17
|
+
{ provider: 'ollama', model: 'qwen2.5-coder:32b' },
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Inspects quota status for a given { provider, model } slot via dsh-key-limits or dsh-subscriptions.
|
|
22
|
+
*
|
|
23
|
+
* @param {object} ctx - Cordis context
|
|
24
|
+
* @param {{ provider: string, model: string }} slot
|
|
25
|
+
* @param {object} [options]
|
|
26
|
+
* @returns {Promise<{ exhausted: boolean, remainingRatio: number, reason: string | null }>}
|
|
27
|
+
*/
|
|
28
|
+
export async function checkSlotQuotaStatus(ctx, slot, options = {}) {
|
|
29
|
+
const provider = String(slot?.provider || '').trim()
|
|
30
|
+
const model = String(slot?.model || '').trim()
|
|
31
|
+
if (!provider) return { exhausted: false, remainingRatio: 1, reason: null }
|
|
32
|
+
|
|
33
|
+
const minRemainingRatio = Number.isFinite(options.minRemainingRatio) ? options.minRemainingRatio : 0.1
|
|
34
|
+
|
|
35
|
+
const keyLimits = options.keyLimitsService || service(ctx, 'keyLimits')
|
|
36
|
+
if (keyLimits) {
|
|
37
|
+
try {
|
|
38
|
+
let status = null
|
|
39
|
+
if (typeof keyLimits.getQuotaStatus === 'function') {
|
|
40
|
+
status = await keyLimits.getQuotaStatus(provider, model)
|
|
41
|
+
} else if (typeof keyLimits.checkLimit === 'function') {
|
|
42
|
+
status = await keyLimits.checkLimit(provider, model)
|
|
43
|
+
} else if (typeof keyLimits.getProviderStatus === 'function') {
|
|
44
|
+
status = await keyLimits.getProviderStatus(provider)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (status) {
|
|
48
|
+
if (status.exhausted === true || status.rateLimited === true || status.blocked === true) {
|
|
49
|
+
return {
|
|
50
|
+
exhausted: true,
|
|
51
|
+
remainingRatio: 0,
|
|
52
|
+
reason: status.reason || 'Quota exhausted or rate-limited in dsh-key-limits',
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const rem = Number(status.remaining ?? status.remainingRequests ?? status.remainingTokens)
|
|
56
|
+
const lim = Number(status.limit ?? status.maxRequests ?? status.maxTokens)
|
|
57
|
+
if (Number.isFinite(rem) && Number.isFinite(lim) && lim > 0) {
|
|
58
|
+
const ratio = rem / lim
|
|
59
|
+
if (ratio < minRemainingRatio) {
|
|
60
|
+
return {
|
|
61
|
+
exhausted: true,
|
|
62
|
+
remainingRatio: ratio,
|
|
63
|
+
reason: `Low quota remaining (${Math.round(ratio * 100)}%) in dsh-key-limits`,
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return { exhausted: false, remainingRatio: ratio, reason: null }
|
|
67
|
+
}
|
|
68
|
+
if (Number.isFinite(status.remainingRatio)) {
|
|
69
|
+
if (status.remainingRatio < minRemainingRatio) {
|
|
70
|
+
return {
|
|
71
|
+
exhausted: true,
|
|
72
|
+
remainingRatio: status.remainingRatio,
|
|
73
|
+
reason: `Low quota ratio (${Math.round(status.remainingRatio * 100)}%) in dsh-key-limits`,
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return { exhausted: false, remainingRatio: status.remainingRatio, reason: null }
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
} catch (err) {
|
|
80
|
+
logger.warn(`[dsh-moa:quota] Error querying dsh-key-limits for ${provider}:${model}:`, err?.message || err)
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const subscriptions = options.subscriptionsService || service(ctx, 'subscriptions')
|
|
85
|
+
if (subscriptions) {
|
|
86
|
+
try {
|
|
87
|
+
let subStatus = null
|
|
88
|
+
if (typeof subscriptions.getSubscriptionStatus === 'function') {
|
|
89
|
+
subStatus = await subscriptions.getSubscriptionStatus(provider, model)
|
|
90
|
+
} else if (typeof subscriptions.checkQuota === 'function') {
|
|
91
|
+
subStatus = await subscriptions.checkQuota(provider, model)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (subStatus) {
|
|
95
|
+
if (subStatus.exhausted === true || subStatus.active === false || subStatus.expired === true) {
|
|
96
|
+
return {
|
|
97
|
+
exhausted: true,
|
|
98
|
+
remainingRatio: 0,
|
|
99
|
+
reason: subStatus.reason || 'Subscription quota exhausted or inactive in dsh-subscriptions',
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (Number.isFinite(subStatus.remainingRatio) && subStatus.remainingRatio < minRemainingRatio) {
|
|
103
|
+
return {
|
|
104
|
+
exhausted: true,
|
|
105
|
+
remainingRatio: subStatus.remainingRatio,
|
|
106
|
+
reason: `Low subscription quota (${Math.round(subStatus.remainingRatio * 100)}%) in dsh-subscriptions`,
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
} catch (err) {
|
|
111
|
+
logger.warn(`[dsh-moa:quota] Error querying dsh-subscriptions for ${provider}:${model}:`, err?.message || err)
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return { exhausted: false, remainingRatio: 1, reason: null }
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Balances candidate reference slots before Layer 1 execution.
|
|
120
|
+
* Preserves quota for the aggregator by substituting low-quota/exhausted slots
|
|
121
|
+
* with healthy fallback models.
|
|
122
|
+
*
|
|
123
|
+
* @param {Array<{ provider: string, model: string }>} references
|
|
124
|
+
* @param {object} options
|
|
125
|
+
* @returns {Promise<{ balancedReferences: Array<{ provider: string, model: string }>, substitutions: Array<object> }>}
|
|
126
|
+
*/
|
|
127
|
+
export async function balanceCandidateQuotas(references = [], options = {}) {
|
|
128
|
+
if (!Array.isArray(references) || references.length === 0) {
|
|
129
|
+
return { balancedReferences: [], substitutions: [] }
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const ctx = options.ctx || null
|
|
133
|
+
const aggregator = options.aggregator || null
|
|
134
|
+
const customFallbacks = Array.isArray(options.fallbackPool) && options.fallbackPool.length > 0
|
|
135
|
+
? options.fallbackPool
|
|
136
|
+
: DEFAULT_FALLBACK_POOL
|
|
137
|
+
|
|
138
|
+
const usedKeys = new Set(references.map((r) => slotLabel(r)))
|
|
139
|
+
const balancedReferences = []
|
|
140
|
+
const substitutions = []
|
|
141
|
+
|
|
142
|
+
for (let i = 0; i < references.length; i++) {
|
|
143
|
+
const slot = references[i]
|
|
144
|
+
const label = slotLabel(slot)
|
|
145
|
+
|
|
146
|
+
// Reserve higher threshold (25%) if this candidate uses the exact same provider:model as the final aggregator
|
|
147
|
+
const isSameAsAggregator = aggregator && slotLabel(aggregator) === label
|
|
148
|
+
const minRemainingRatio = isSameAsAggregator ? 0.25 : (options.minRemainingRatio ?? 0.1)
|
|
149
|
+
|
|
150
|
+
const status = await checkSlotQuotaStatus(ctx, slot, {
|
|
151
|
+
...options,
|
|
152
|
+
minRemainingRatio,
|
|
153
|
+
})
|
|
154
|
+
|
|
155
|
+
if (!status.exhausted) {
|
|
156
|
+
balancedReferences.push(slot)
|
|
157
|
+
continue
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Find a healthy replacement from customFallbacks
|
|
161
|
+
let replacement = null
|
|
162
|
+
for (const candidateFallback of customFallbacks) {
|
|
163
|
+
const fbLabel = slotLabel(candidateFallback)
|
|
164
|
+
if (fbLabel === label) continue
|
|
165
|
+
const fbStatus = await checkSlotQuotaStatus(ctx, candidateFallback, options)
|
|
166
|
+
if (!fbStatus.exhausted) {
|
|
167
|
+
replacement = { ...slot, provider: candidateFallback.provider, model: candidateFallback.model, replacedFrom: label }
|
|
168
|
+
usedKeys.add(fbLabel)
|
|
169
|
+
break
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (replacement) {
|
|
174
|
+
balancedReferences.push(replacement)
|
|
175
|
+
substitutions.push({
|
|
176
|
+
index: i + 1,
|
|
177
|
+
original: slot,
|
|
178
|
+
replacement: { provider: replacement.provider, model: replacement.model },
|
|
179
|
+
reason: status.reason,
|
|
180
|
+
})
|
|
181
|
+
logger.info(`[dsh-moa:quota] Replaced slot #${i + 1} (${label}) -> ${slotLabel(replacement)} (${status.reason})`)
|
|
182
|
+
} else {
|
|
183
|
+
balancedReferences.push(slot)
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
return { balancedReferences, substitutions }
|
|
188
|
+
}
|
package/lib/moa-schema.js
CHANGED
|
@@ -66,6 +66,10 @@ export const PresetSchema = z.object({
|
|
|
66
66
|
uncertainty_scoring_enabled: z.boolean().default(false),
|
|
67
67
|
subagent_swarm_enabled: z.boolean().default(false),
|
|
68
68
|
adaptive_personas_enabled: z.boolean().default(false),
|
|
69
|
+
quota_aware_balancing_enabled: z.boolean().default(false),
|
|
70
|
+
self_healing_enabled: z.boolean().default(false),
|
|
71
|
+
self_healing_max_retries: z.number().default(2),
|
|
72
|
+
self_healing_command: z.string().default("npm test"),
|
|
69
73
|
})
|
|
70
74
|
|
|
71
75
|
export const Config = z.object({
|
|
@@ -78,6 +82,10 @@ export const Config = z.object({
|
|
|
78
82
|
uncertainty_scoring_enabled: z.boolean().default(false).volatile(),
|
|
79
83
|
subagent_swarm_enabled: z.boolean().default(false).volatile(),
|
|
80
84
|
adaptive_personas_enabled: z.boolean().default(false).volatile(),
|
|
85
|
+
quota_aware_balancing_enabled: z.boolean().default(false).volatile(),
|
|
86
|
+
self_healing_enabled: z.boolean().default(false).volatile(),
|
|
87
|
+
self_healing_max_retries: z.number().default(2).volatile(),
|
|
88
|
+
self_healing_command: z.string().default("npm test").volatile(),
|
|
81
89
|
smart_routing_model: ModelSlotSchema.default({ provider: '', model: '' }).volatile(),
|
|
82
90
|
prices: z.dict(PriceRow).default({}).volatile(),
|
|
83
91
|
presets: z.array(PresetSchema).default([
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Self-Healing Post-Promotion Build Loop with dsh-time-machine Auto-Rollback (#252)
|
|
3
|
+
* Executes verification command after file promotion, dispatches compiler/test error logs
|
|
4
|
+
* to the MoA aggregator for hotfix generation up to maxRetries times, and automatically
|
|
5
|
+
* rolls back to the pre-promotion dsh-time-machine snapshot if verification still fails.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { execFile } from 'node:child_process'
|
|
9
|
+
import { promisify } from 'node:util'
|
|
10
|
+
import fs from 'node:fs/promises'
|
|
11
|
+
import path from 'node:path'
|
|
12
|
+
import { logger } from './logger.js'
|
|
13
|
+
import { bestEffort } from './best-effort.js'
|
|
14
|
+
import { extractFileBlocks } from './workspace/fs.js'
|
|
15
|
+
import { isSafeRelativePath, assertPathContained, isLoopbackUrl } from './workspace/security.js'
|
|
16
|
+
|
|
17
|
+
const execFileAsync = promisify(execFile)
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Runs a verification command in the target workspace.
|
|
21
|
+
*
|
|
22
|
+
* @param {string} cwd
|
|
23
|
+
* @param {string} commandStr
|
|
24
|
+
* @param {object} [options]
|
|
25
|
+
* @returns {Promise<{ passed: boolean, exitCode: number, output: string }>}
|
|
26
|
+
*/
|
|
27
|
+
export async function runVerificationCommand(cwd, commandStr = 'npm test', options = {}) {
|
|
28
|
+
if (typeof options.execFn === 'function') {
|
|
29
|
+
return await options.execFn(cwd, commandStr)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const trimmed = String(commandStr || 'npm test').trim()
|
|
33
|
+
const isWin = process.platform === 'win32'
|
|
34
|
+
const shell = isWin ? 'cmd.exe' : '/bin/sh'
|
|
35
|
+
const shellArgs = isWin ? ['/d', '/s', '/c', trimmed] : ['-c', trimmed]
|
|
36
|
+
|
|
37
|
+
try {
|
|
38
|
+
const { stdout, stderr } = await execFileAsync(shell, shellArgs, {
|
|
39
|
+
cwd,
|
|
40
|
+
timeout: (options.timeoutSec ?? 60) * 1000,
|
|
41
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
42
|
+
signal: options.signal,
|
|
43
|
+
})
|
|
44
|
+
return {
|
|
45
|
+
passed: true,
|
|
46
|
+
exitCode: 0,
|
|
47
|
+
output: `${stdout || ''}\n${stderr || ''}`.trim(),
|
|
48
|
+
}
|
|
49
|
+
} catch (err) {
|
|
50
|
+
const out = `${err?.stdout || ''}\n${err?.stderr || ''}\n${err?.message || ''}`.trim()
|
|
51
|
+
return {
|
|
52
|
+
passed: false,
|
|
53
|
+
exitCode: typeof err?.code === 'number' ? err.code : 1,
|
|
54
|
+
output: out.slice(0, 12000),
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Triggers an automatic rollback to a pre-promotion dsh-time-machine snapshot.
|
|
61
|
+
*
|
|
62
|
+
* @param {string} baseDir
|
|
63
|
+
* @param {object} snapshot
|
|
64
|
+
* @param {object} [options]
|
|
65
|
+
* @returns {Promise<{ rolledBack: boolean, snapshotId: string | null }>}
|
|
66
|
+
*/
|
|
67
|
+
export async function rollbackToTimeMachineSnapshot(baseDir, snapshot, options = {}) {
|
|
68
|
+
const snapshotId = typeof snapshot === 'string'
|
|
69
|
+
? snapshot
|
|
70
|
+
: (snapshot?.id || snapshot?.snapshotId || snapshot?.commit || snapshot?.hash || null)
|
|
71
|
+
|
|
72
|
+
if (typeof options.rollbackFn === 'function') {
|
|
73
|
+
const res = await bestEffort('custom rollbackFn', () => options.rollbackFn(baseDir, snapshotId, snapshot))
|
|
74
|
+
return { rolledBack: Boolean(res), snapshotId }
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (options.timeMachineEngine) {
|
|
78
|
+
const fn = options.timeMachineEngine.restoreSnapshot || options.timeMachineEngine.rollback
|
|
79
|
+
if (typeof fn === 'function') {
|
|
80
|
+
const res = await bestEffort('dsh-time-machine direct restoreSnapshot', () =>
|
|
81
|
+
fn.call(options.timeMachineEngine, snapshotId, { cwd: baseDir })
|
|
82
|
+
)
|
|
83
|
+
if (res) return { rolledBack: true, snapshotId }
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const port = options.port || options.webServerPort || process.env.DSH_PORT || process.env.PORT
|
|
88
|
+
if (!port && !options.timeMachineRollbackUrl) {
|
|
89
|
+
logger.warn('[dsh-moa:self-healing] Cannot rollback: webServerPort is unknown and no timeMachineEngine/rollbackFn provided')
|
|
90
|
+
return { rolledBack: false, snapshotId }
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const endpoint = options.timeMachineRollbackUrl || `http://127.0.0.1:${port}/dsh-time-machine/restore`
|
|
94
|
+
if (!isLoopbackUrl(endpoint)) {
|
|
95
|
+
logger.warn(`[dsh-moa:self-healing] Blocked non-loopback rollback URL: ${endpoint}`)
|
|
96
|
+
return { rolledBack: false, snapshotId }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const restored = await bestEffort('dsh-time-machine HTTP rollback', async () => {
|
|
100
|
+
const res = await fetch(endpoint, {
|
|
101
|
+
method: 'POST',
|
|
102
|
+
headers: { 'Content-Type': 'application/json' },
|
|
103
|
+
body: JSON.stringify({ id: snapshotId, snapshotId, cwd: baseDir }),
|
|
104
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? 5000),
|
|
105
|
+
redirect: 'error',
|
|
106
|
+
})
|
|
107
|
+
return res.ok
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
return { rolledBack: Boolean(restored), snapshotId }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Applies extracted hotfix files directly to the workspace directory.
|
|
115
|
+
*/
|
|
116
|
+
async function applyHotfixFiles(cwd, files = []) {
|
|
117
|
+
const written = []
|
|
118
|
+
for (const f of files) {
|
|
119
|
+
const rel = f.relativePath || f.path
|
|
120
|
+
if (!rel || !isSafeRelativePath(rel)) continue
|
|
121
|
+
const targetPath = assertPathContained(cwd, rel)
|
|
122
|
+
await fs.mkdir(path.dirname(targetPath), { recursive: true })
|
|
123
|
+
await fs.writeFile(targetPath, f.content, 'utf8')
|
|
124
|
+
written.push(rel)
|
|
125
|
+
}
|
|
126
|
+
return written
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Runs the post-promotion self-healing build loop.
|
|
131
|
+
*
|
|
132
|
+
* @param {object} params
|
|
133
|
+
* @returns {Promise<{
|
|
134
|
+
* enabled: boolean,
|
|
135
|
+
* passed: boolean,
|
|
136
|
+
* attempts: number,
|
|
137
|
+
* rolledBack: boolean,
|
|
138
|
+
* snapshotId: string | null,
|
|
139
|
+
* hotfixFiles: string[],
|
|
140
|
+
* lastOutput: string
|
|
141
|
+
* }>}
|
|
142
|
+
*/
|
|
143
|
+
export async function runSelfHealingLoop({
|
|
144
|
+
cwd,
|
|
145
|
+
command = 'npm test',
|
|
146
|
+
maxRetries = 2,
|
|
147
|
+
aggregatorSlot,
|
|
148
|
+
callLlm,
|
|
149
|
+
userPrompt = '',
|
|
150
|
+
promotedFiles = [],
|
|
151
|
+
checkpoint = null,
|
|
152
|
+
promoteOpts = {},
|
|
153
|
+
onProgress = null,
|
|
154
|
+
signal = null,
|
|
155
|
+
execFn = null,
|
|
156
|
+
rollbackFn = null,
|
|
157
|
+
}) {
|
|
158
|
+
const retries = Math.max(0, Math.min(5, Number(maxRetries ?? 2)))
|
|
159
|
+
const hotfixFiles = new Set()
|
|
160
|
+
|
|
161
|
+
if (typeof onProgress === 'function') {
|
|
162
|
+
onProgress(`🩺 *Self-Healing Verification: Running \`${command}\`...*\n`)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
let check = await runVerificationCommand(cwd, command, { execFn, signal })
|
|
166
|
+
if (check.passed) {
|
|
167
|
+
return {
|
|
168
|
+
enabled: true,
|
|
169
|
+
passed: true,
|
|
170
|
+
attempts: 0,
|
|
171
|
+
rolledBack: false,
|
|
172
|
+
snapshotId: null,
|
|
173
|
+
hotfixFiles: [],
|
|
174
|
+
lastOutput: check.output,
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
let attempts = 0
|
|
179
|
+
while (!check.passed && attempts < retries) {
|
|
180
|
+
if (signal?.aborted) break
|
|
181
|
+
attempts++
|
|
182
|
+
|
|
183
|
+
if (typeof onProgress === 'function') {
|
|
184
|
+
onProgress(`🔧 *Self-Healing Attempt ${attempts}/${retries}: Requesting hotfix patch from aggregator...*\n`)
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const fileList = Array.isArray(promotedFiles)
|
|
188
|
+
? promotedFiles.map((f) => (typeof f === 'string' ? f : f.relativePath || f.path)).filter(Boolean)
|
|
189
|
+
: []
|
|
190
|
+
|
|
191
|
+
const hotfixPrompt = [
|
|
192
|
+
`The promoted MoA changes failed the verification command \`${command}\` (exit code ${check.exitCode}).`,
|
|
193
|
+
`Original User Task: ${userPrompt}`,
|
|
194
|
+
`Promoted Files: ${fileList.join(', ') || 'workspace files'}`,
|
|
195
|
+
`\n### Compiler / Test Failure Output:\n\`\`\`\n${check.output}\n\`\`\``,
|
|
196
|
+
`\nProvide a minimal hotfix patch using markdown code blocks with \`path=<relative/path>\` header so the build and tests pass.`,
|
|
197
|
+
].join('\n')
|
|
198
|
+
|
|
199
|
+
const llmRes = await callLlm({
|
|
200
|
+
provider: aggregatorSlot?.provider || 'default',
|
|
201
|
+
model: aggregatorSlot?.model || 'default',
|
|
202
|
+
messages: [
|
|
203
|
+
{
|
|
204
|
+
role: 'system',
|
|
205
|
+
content: 'You are an expert automated build healer. Fix compilation or unit test errors by outputting complete corrected files with ```lang path=relative/path.ext fences.',
|
|
206
|
+
},
|
|
207
|
+
{ role: 'user', content: hotfixPrompt },
|
|
208
|
+
],
|
|
209
|
+
temperature: 0.2,
|
|
210
|
+
maxTokens: 4096,
|
|
211
|
+
signal,
|
|
212
|
+
})
|
|
213
|
+
|
|
214
|
+
const patchText = typeof llmRes === 'string' ? llmRes : (llmRes?.content || llmRes?.text || '')
|
|
215
|
+
const extracted = extractFileBlocks(patchText)
|
|
216
|
+
if (extracted.length > 0) {
|
|
217
|
+
const applied = await applyHotfixFiles(cwd, extracted)
|
|
218
|
+
for (const f of applied) hotfixFiles.add(f)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
check = await runVerificationCommand(cwd, command, { execFn, signal })
|
|
222
|
+
if (check.passed) {
|
|
223
|
+
if (typeof onProgress === 'function') {
|
|
224
|
+
onProgress(`✅ *Self-Healing Succeeded on attempt ${attempts}/${retries}!*\n`)
|
|
225
|
+
}
|
|
226
|
+
return {
|
|
227
|
+
enabled: true,
|
|
228
|
+
passed: true,
|
|
229
|
+
attempts,
|
|
230
|
+
rolledBack: false,
|
|
231
|
+
snapshotId: null,
|
|
232
|
+
hotfixFiles: [...hotfixFiles],
|
|
233
|
+
lastOutput: check.output,
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// Verification still failed after maxRetries -> unconditional Time Machine rollback
|
|
239
|
+
const snapshotObj = checkpoint || promoteOpts?.checkpointSnapshot || (Array.isArray(promotedFiles) && promotedFiles._checkpoint) || null
|
|
240
|
+
if (typeof onProgress === 'function') {
|
|
241
|
+
onProgress(`⏪ *Self-Healing exhausted (${attempts}/${retries}). Rolling back workspace via dsh-time-machine...*\n`)
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const rb = await rollbackToTimeMachineSnapshot(cwd, snapshotObj, {
|
|
245
|
+
...promoteOpts,
|
|
246
|
+
rollbackFn,
|
|
247
|
+
})
|
|
248
|
+
|
|
249
|
+
return {
|
|
250
|
+
enabled: true,
|
|
251
|
+
passed: false,
|
|
252
|
+
attempts,
|
|
253
|
+
rolledBack: rb.rolledBack,
|
|
254
|
+
snapshotId: rb.snapshotId,
|
|
255
|
+
hotfixFiles: [...hotfixFiles],
|
|
256
|
+
lastOutput: check.output,
|
|
257
|
+
}
|
|
258
|
+
}
|
package/lib/moa-workbench.js
CHANGED
|
@@ -27,6 +27,18 @@ export const COMPANION_PLUGINS = [
|
|
|
27
27
|
url: 'https://github.com/goodandready/dsh-time-machine',
|
|
28
28
|
description: 'Pre-promotion shadow Git checkpoints and rollback points',
|
|
29
29
|
},
|
|
30
|
+
{
|
|
31
|
+
id: 'dsh-key-limits',
|
|
32
|
+
name: 'DSH Key Limits',
|
|
33
|
+
url: 'https://github.com/GooDAnDReaDY/dsh-key-limits',
|
|
34
|
+
description: 'Provider API key rate-limit and quota telemetry',
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
id: 'dsh-subscriptions',
|
|
38
|
+
name: 'DSH Subscriptions',
|
|
39
|
+
url: 'https://github.com/GooDAnDReaDY/dsh-subscriptions',
|
|
40
|
+
description: 'Subscription tier and quota tracking for LLM providers',
|
|
41
|
+
},
|
|
30
42
|
]
|
|
31
43
|
|
|
32
44
|
/**
|
package/lib/pipeline/context.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { balanceCandidateQuotas } from '../moa-quota-balancer.js'
|
|
1
2
|
import { collectProjectContext, formatProjectContext, isRefinementTask } from '../workspace/scanner.js'
|
|
2
3
|
import { applyBudgetGuardrails, createBudgetAbortPayload } from '../moa-budget.js'
|
|
3
4
|
import { extractPriorTurnBaseline, pruneMultiTurnMessages } from '../moa-context.js'
|
|
@@ -13,6 +14,7 @@ export async function stageProjectContext({
|
|
|
13
14
|
maxTokens,
|
|
14
15
|
startTime,
|
|
15
16
|
onProgress,
|
|
17
|
+
ctx = null,
|
|
16
18
|
}) {
|
|
17
19
|
const rawRefs = (Array.isArray(preset?.reference_models) && preset.reference_models.length > 0)
|
|
18
20
|
? preset.reference_models
|
|
@@ -79,7 +81,23 @@ export async function stageProjectContext({
|
|
|
79
81
|
return { ok: false, abortPayload }
|
|
80
82
|
}
|
|
81
83
|
|
|
82
|
-
|
|
84
|
+
let effectiveReferences = budgetCheck.references
|
|
85
|
+
let quotaSubstitutions = []
|
|
86
|
+
if (preset?.quota_aware_balancing_enabled) {
|
|
87
|
+
const fallbackPool = Array.isArray(preset?.local_fallback_models) && preset.local_fallback_models.length > 0
|
|
88
|
+
? preset.local_fallback_models
|
|
89
|
+
: undefined
|
|
90
|
+
const bal = await balanceCandidateQuotas(effectiveReferences, {
|
|
91
|
+
ctx,
|
|
92
|
+
aggregator: primaryJudge,
|
|
93
|
+
fallbackPool,
|
|
94
|
+
})
|
|
95
|
+
effectiveReferences = bal.balancedReferences
|
|
96
|
+
quotaSubstitutions = bal.substitutions
|
|
97
|
+
if (quotaSubstitutions.length > 0 && typeof onProgress === "function") {
|
|
98
|
+
onProgress(`⚖️ *Quota-Aware Balancing: Replaced ${quotaSubstitutions.length} low-quota slot(s) to preserve aggregator budget.*\n`)
|
|
99
|
+
}
|
|
100
|
+
}
|
|
83
101
|
if (effectiveReferences.length === 1 && !preset?.curator_synthesis) {
|
|
84
102
|
isFastMode = true
|
|
85
103
|
}
|
|
@@ -103,7 +121,7 @@ export async function stageProjectContext({
|
|
|
103
121
|
isStreamAggregator, isQuorumEnabled, gracePeriodSec, candidateRetries,
|
|
104
122
|
refTimeoutSec, aggTimeoutSec, isBlindEvaluation, isPeerCritiqueEnabled,
|
|
105
123
|
allowCandidateOverride, collectedCtx, isRefinement, projectContext,
|
|
106
|
-
effectiveReferences, priorTurnBaseline, candidateSystemPrompt,
|
|
124
|
+
effectiveReferences, quotaSubstitutions, priorTurnBaseline, candidateSystemPrompt,
|
|
107
125
|
needsQuestions, enrichedMessages,
|
|
108
126
|
}
|
|
109
127
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { runSelfHealingLoop } from '../moa-self-healing.js'
|
|
1
2
|
import { logger } from '../logger.js'
|
|
2
3
|
import { parseWinnerIndex, parseRecommendedAssembler } from '../moa-parser.js'
|
|
3
4
|
import { extractFileBlocks, writeCandidateWorkspace, promoteCandidateWorkspace, cleanMoaWorkspaces } from '../workspace/fs.js'
|
|
@@ -33,6 +34,9 @@ export async function stagePromotion({
|
|
|
33
34
|
historyFilePath,
|
|
34
35
|
collectedCtx,
|
|
35
36
|
ctx = null,
|
|
37
|
+
callLlm = null,
|
|
38
|
+
onProgress = null,
|
|
39
|
+
execFn = null,
|
|
36
40
|
}) {
|
|
37
41
|
const winningIndex = (multiJudgeResult?.consensus && multiJudgeResult?.winningCandidateIndex)
|
|
38
42
|
? multiJudgeResult.winningCandidateIndex
|
|
@@ -69,6 +73,24 @@ export async function stagePromotion({
|
|
|
69
73
|
throw promoteErr
|
|
70
74
|
}
|
|
71
75
|
|
|
76
|
+
let selfHealingResult = null
|
|
77
|
+
if (preset?.self_healing_enabled && promotedFiles.length > 0 && typeof callLlm === "function") {
|
|
78
|
+
selfHealingResult = await runSelfHealingLoop({
|
|
79
|
+
cwd,
|
|
80
|
+
command: preset.self_healing_command || "npm test",
|
|
81
|
+
maxRetries: preset.self_healing_max_retries ?? 2,
|
|
82
|
+
aggregatorSlot: chosenJudge,
|
|
83
|
+
callLlm,
|
|
84
|
+
userPrompt,
|
|
85
|
+
promotedFiles,
|
|
86
|
+
checkpoint: promotedFiles.checkpoint || null,
|
|
87
|
+
promoteOpts,
|
|
88
|
+
onProgress,
|
|
89
|
+
signal,
|
|
90
|
+
execFn,
|
|
91
|
+
})
|
|
92
|
+
}
|
|
93
|
+
|
|
72
94
|
const livePreview = await createPromotedPreview(liveCanvas, cwd, promotedFiles, referenceOutputs, { winningIndex, synthesizedFiles })
|
|
73
95
|
const durationMs = Date.now() - startTime
|
|
74
96
|
const usageSummary = summarizeMoAUsage(referenceOutputs, aggUsage, multiJudgeResult?.totalUsage)
|
|
@@ -125,7 +147,7 @@ export async function stagePromotion({
|
|
|
125
147
|
kind: 'synthesis', content: synthesizedText, aggregator: isFastMode ? 'Fast Mode (Direct)' : finalAggLabel,
|
|
126
148
|
references: referenceOutputs, presetName: preset?.name || 'default', isRefinement, isFastMode, isCuratorSynthesis,
|
|
127
149
|
recommendedAssembler, winningIndex, winnerModel, promotedFiles, runId, allowCandidateOverride,
|
|
128
|
-
benchmarkReport, consensus: multiJudgeResult, fileScores, ...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
150
|
+
benchmarkReport, consensus: multiJudgeResult, fileScores, ...(selfHealingResult ? { selfHealing: selfHealingResult } : {}), ...(livePreview ? { liveCanvas: livePreview } : {}),
|
|
129
151
|
usage: usageSummary, skippedFiles: collectedCtx?.skippedFiles || 0, skippedList: collectedCtx?.skippedList || [], durationMs,
|
|
130
152
|
}
|
|
131
153
|
}
|
package/lib/pipeline/runner.js
CHANGED
|
@@ -40,7 +40,7 @@ export async function runMoAPipeline({
|
|
|
40
40
|
|
|
41
41
|
// Stage 1: Context & Configuration
|
|
42
42
|
const ctxStage = await stageProjectContext({
|
|
43
|
-
userPrompt, messages, preset, cwd, prices, maxTokens: preset?.max_tokens, startTime, onProgress,
|
|
43
|
+
userPrompt, messages, preset, cwd, prices, maxTokens: preset?.max_tokens, startTime, onProgress, ctx,
|
|
44
44
|
})
|
|
45
45
|
if (!ctxStage.ok) return ctxStage.abortPayload
|
|
46
46
|
|
|
@@ -189,7 +189,7 @@ export async function runMoAPipeline({
|
|
|
189
189
|
promoteOpts, cwd, runId, signal, liveCanvas, startTime, chosenJudge, aggUsage,
|
|
190
190
|
userPrompt, preset, isRefinement: ctxStage.isRefinement, isFastMode: ctxStage.isFastMode,
|
|
191
191
|
effectiveReferences: ctxStage.effectiveReferences, referenceModels: ctxStage.referenceModels,
|
|
192
|
-
historyFilePath, collectedCtx: ctxStage.collectedCtx, ctx,
|
|
192
|
+
historyFilePath, collectedCtx: ctxStage.collectedCtx, ctx, callLlm, onProgress, execFn,
|
|
193
193
|
})
|
|
194
194
|
if (!promRes.ok) {
|
|
195
195
|
if (promRes.abort) return { error: promRes.error }
|