@goodandready/dsh-moa 0.2.78 → 0.2.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/README.ru.md +2 -0
- package/README.zh.md +2 -0
- package/lib/client.js +110 -2
- package/lib/events.js +2 -0
- package/lib/moa-schema.js +10 -0
- package/lib/pipeline/budget-guard.js +158 -0
- package/lib/pipeline/pareto-frontier.js +167 -0
- package/lib/pipeline/runner.js +30 -2
- package/lib/pricing.js +8 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -58,6 +58,8 @@ Single-model AI generation often suffers from blind spots, single-perspective bi
|
|
|
58
58
|
15. **Mutation Testing Gate for Candidate Tests (#316)**: Injects AST-level fault mutations to compute Mutation Score (killed vs survived mutants), weeding out tautological assertions and fake test suites.
|
|
59
59
|
16. **Hybrid Private Edge MoA & Cloud Judge Sanitizer (#306)**: Preserves confidentiality with local edge proposers on MiniPC while scrubbing absolute paths, internal IPs, and API secrets before external Cloud Judge arbitration.
|
|
60
60
|
17. **Heterogeneous Speculative Decoding (#317)**: High-throughput local edge draft coupled with single-pass cloud model verification, slashing response latency by 2-3x.
|
|
61
|
+
18. **Cost/Quality Pareto Frontier Slider (#307)**: Interactive trade-off slider mapping cost vs quality along the Pareto Frontier with real-time turn cost & latency estimation.
|
|
62
|
+
19. **Hard Token Budget Guard (#319)**: Pre-flight financial cost ceiling with dynamic auto-scaling of cloud models to local edge to guarantee turn budgets.
|
|
61
63
|
|
|
62
64
|
---
|
|
63
65
|
|
package/README.ru.md
CHANGED
|
@@ -58,6 +58,8 @@
|
|
|
58
58
|
15. **Мутационное тестирование тестов кандидатов (#316)**: Внедрение AST-мутаций кода для расчёта Mutation Score (убитые vs выжившие мутанты), устраняющее ложноположительные и тавтологические тесты.
|
|
59
59
|
16. **Гибридный приватный Edge MoA (#306)**: Локальные генераторы на MiniPC сохраняют секреты и кодовую базу в контуре, маскируя пути, IP и токены перед отправкой судье в облако.
|
|
60
60
|
17. **Гетерогенное спекулятивное декодирование (#317)**: Быстрый черновик от локальной edge-модели с пакетной валидацией и доработкой флагманом в облаке с ускорением в 2-3 раза.
|
|
61
|
+
18. **Слайдер фронтира Парето (Стоимость vs Качество) (#307)**: Интерактивный регулятор баланса затрат и качества консенсуса с реактивным прогнозом цены и задержки.
|
|
62
|
+
19. **Защитный барьер бюджета токенов (#319)**: Строгий финансовый лимит стоимости сессии с динамическим авто-масштабированием и заменой облачных слотов на локальные.
|
|
61
63
|
|
|
62
64
|
---
|
|
63
65
|
|
package/README.zh.md
CHANGED
|
@@ -58,6 +58,8 @@
|
|
|
58
58
|
15. **候选测试套件变异测试关卡 (#316)**:注入 AST 级别故障变异以计算变异得分(消灭 vs 存活变异体),剔除恒真废测试与虚假测试套件。
|
|
59
59
|
16. **混合隐私边缘 MoA 与云端裁判脱敏 (#306)**:本地 MiniPC 边缘模型在内网读取代码生成方案,在提交给外部云端裁判前自动脱敏绝对路径、内网 IP 与 API 密钥。
|
|
60
60
|
17. **异构投机解码加速 (#317)**:本地小模型快速生成投机代码草稿块,云端旗舰模型单次批处理验证与完善,首包延迟降低 2-3 倍。
|
|
61
|
+
18. **成本与质量帕累托前沿滑块 (#307)**:互动式调节成本、质量与延迟的帕累托平衡,实时预估单轮开销与耗时。
|
|
62
|
+
19. **刚性 Token 预算保护与动态缩放 (#319)**:发送前严守费用上限,超出时自动将昂贵云端模型降级为本地边缘模型以确保不超支。
|
|
61
63
|
|
|
62
64
|
---
|
|
63
65
|
|
package/lib/client.js
CHANGED
|
@@ -240,7 +240,16 @@ window.__ModuleLoader__.load({
|
|
|
240
240
|
"aggregator.privacy_gate_title": "\u{1F6E1}\uFE0F Hybrid Private Edge MoA (Local Edge Proposers + Cloud Judge)",
|
|
241
241
|
"aggregator.privacy_gate_desc": "Scans and redacts proprietary secrets, local paths, and private network IPs before forwarding proposals to external cloud judges.",
|
|
242
242
|
"aggregator.speculative_gate_title": "\u26A1 Heterogeneous Speculative Decoding (Local Draft + Cloud Verifier)",
|
|
243
|
-
"aggregator.speculative_gate_desc": "Lightweight local edge model generates high-speed token drafts verified and completed in one fast pass by the cloud model."
|
|
243
|
+
"aggregator.speculative_gate_desc": "Lightweight local edge model generates high-speed token drafts verified and completed in one fast pass by the cloud model.",
|
|
244
|
+
"aggregator.pareto_slider_title": "\u{1F4C8} Cost vs Quality Pareto Frontier Slider",
|
|
245
|
+
"aggregator.pareto_slider_desc": "Reactive balance from 100% free local edge models to ultra-rigorous flagship multi-judge consensus.",
|
|
246
|
+
"aggregator.pareto_pos_label": "Frontier Position:",
|
|
247
|
+
"aggregator.hard_budget_title": "\u{1F6E1}\uFE0F Hard Token Budget Guard with Auto-Scaling",
|
|
248
|
+
"aggregator.hard_budget_desc": "Pre-calculates turn costs against a hard financial ceiling, automatically scaling down or halting execution.",
|
|
249
|
+
"aggregator.hard_budget_limit_label": "Max Cost Per Turn (USD):",
|
|
250
|
+
"aggregator.hard_budget_policy_label": "Exceed Policy:",
|
|
251
|
+
"aggregator.hard_budget_policy_downgrade": "Auto-Downgrade to Local/Economic",
|
|
252
|
+
"aggregator.hard_budget_policy_fail": "Fail-Closed (Halt Execution)"
|
|
244
253
|
};
|
|
245
254
|
const zh = {
|
|
246
255
|
title: "\u667A\u80FD\u4F53\u6DF7\u5408 (MoA)",
|
|
@@ -471,7 +480,16 @@ window.__ModuleLoader__.load({
|
|
|
471
480
|
"aggregator.privacy_gate_title": "\u{1F6E1}\uFE0F \u6DF7\u5408\u9690\u79C1\u8FB9\u7F18 MoA (\u672C\u5730\u751F\u6210 + \u4E91\u7AEF\u88C1\u5224)",
|
|
472
481
|
"aggregator.privacy_gate_desc": "\u5728\u5C06\u5019\u9009\u63D0\u6848\u53D1\u9001\u7ED9\u5916\u90E8\u4E91\u7AEF\u88C1\u5224\u4E4B\u524D\uFF0C\u626B\u63CF\u5E76\u8131\u654F\u4E13\u6709\u5BC6\u94A5\u3001\u672C\u5730\u7EDD\u5BF9\u8DEF\u5F84\u4E0E\u79C1\u6709\u7F51\u7EDC IP\u3002",
|
|
473
482
|
"aggregator.speculative_gate_title": "\u26A1 \u5F02\u6784\u6295\u673A\u89E3\u7801 (\u672C\u5730\u8349\u7A3F + \u4E91\u7AEF\u9A8C\u8BC1)",
|
|
474
|
-
"aggregator.speculative_gate_desc": "\u8F7B\u91CF\u7EA7\u672C\u5730\u8FB9\u7F18\u6A21\u578B\u751F\u6210\u9AD8\u901F\u8349\u7A3F\uFF0C\u7531\u5F3A\u5927\u7684\u4E91\u7AEF\u6A21\u578B\u4E00\u6B21\u6027\u5FEB\u901F\u9A8C\u8BC1\u5E76\u5B8C\u5584\u3002"
|
|
483
|
+
"aggregator.speculative_gate_desc": "\u8F7B\u91CF\u7EA7\u672C\u5730\u8FB9\u7F18\u6A21\u578B\u751F\u6210\u9AD8\u901F\u8349\u7A3F\uFF0C\u7531\u5F3A\u5927\u7684\u4E91\u7AEF\u6A21\u578B\u4E00\u6B21\u6027\u5FEB\u901F\u9A8C\u8BC1\u5E76\u5B8C\u5584\u3002",
|
|
484
|
+
"aggregator.pareto_slider_title": "\u{1F4C8} \u6210\u672C\u4E0E\u8D28\u91CF\u5E15\u7D2F\u6258\u524D\u6CBF\u6ED1\u5757 (Pareto Frontier)",
|
|
485
|
+
"aggregator.pareto_slider_desc": "\u4ECE 100% \u514D\u8D39\u672C\u5730\u8FB9\u7F18\u6A21\u578B\u5230\u8D85\u4E25\u8C28\u65D7\u8230\u591A\u88C1\u5224\u5171\u8BC6\u7684\u54CD\u5E94\u5F0F\u5E73\u8861\u8C03\u8282\u3002",
|
|
486
|
+
"aggregator.pareto_pos_label": "\u524D\u6CBF\u5E73\u8861\u4F4D\u7F6E:",
|
|
487
|
+
"aggregator.hard_budget_title": "\u{1F6E1}\uFE0F \u521A\u6027 Token \u9884\u7B97\u4FDD\u62A4\u4E0E\u52A8\u6001\u964D\u7EA7\u7F29\u653E",
|
|
488
|
+
"aggregator.hard_budget_desc": "\u5728\u53D1\u9001\u524D\u4E25\u683C\u6838\u7B97\u6574\u8F6E\u9884\u6D4B\u8D39\u7528\uFF0C\u81EA\u52A8\u964D\u7EA7\u81F3\u7ECF\u6D4E/\u672C\u5730\u6A21\u578B\u6216\u963B\u6B62\u8D85\u51FA\u9884\u7B97\u3002",
|
|
489
|
+
"aggregator.hard_budget_limit_label": "\u5355\u8F6E\u9884\u7B97\u4E0A\u9650 (USD):",
|
|
490
|
+
"aggregator.hard_budget_policy_label": "\u8D85\u989D\u7B56\u7565:",
|
|
491
|
+
"aggregator.hard_budget_policy_downgrade": "\u81EA\u52A8\u964D\u7EA7\u81F3\u672C\u5730/\u7ECF\u6D4E\u6A21\u578B",
|
|
492
|
+
"aggregator.hard_budget_policy_fail": "\u5B89\u5168\u7194\u65AD (\u505C\u6B62\u6267\u884C)"
|
|
475
493
|
};
|
|
476
494
|
function makeT(dict, fallback) {
|
|
477
495
|
return function t(key, vars) {
|
|
@@ -2760,6 +2778,96 @@ window.__ModuleLoader__.load({
|
|
|
2760
2778
|
t("aggregator.speculative_gate_desc") || "Lightweight local edge model generates high-speed token drafts verified and completed in one fast pass by the cloud model."
|
|
2761
2779
|
)
|
|
2762
2780
|
),
|
|
2781
|
+
/* Pareto Frontier Slider (#307) */
|
|
2782
|
+
React.createElement(
|
|
2783
|
+
"div",
|
|
2784
|
+
{ style: { display: "flex", flexDirection: "column", gap: 6 } },
|
|
2785
|
+
React.createElement(
|
|
2786
|
+
"label",
|
|
2787
|
+
{ style: { display: "flex", alignItems: "center", gap: 8, fontSize: 13, fontWeight: 600, color: "var(--dsw-alias-label-primary)", cursor: "pointer" } },
|
|
2788
|
+
React.createElement("input", {
|
|
2789
|
+
type: "checkbox",
|
|
2790
|
+
checked: Boolean(currentPreset.pareto_slider_enabled),
|
|
2791
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, pareto_slider_enabled: e.target.checked })),
|
|
2792
|
+
style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
|
|
2793
|
+
}),
|
|
2794
|
+
t("aggregator.pareto_slider_title") || "\u{1F4C8} Cost vs Quality Pareto Frontier Slider"
|
|
2795
|
+
),
|
|
2796
|
+
React.createElement(
|
|
2797
|
+
"div",
|
|
2798
|
+
{ style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
|
|
2799
|
+
t("aggregator.pareto_slider_desc") || "Reactive balance from 100% free local edge models to ultra-rigorous flagship multi-judge consensus."
|
|
2800
|
+
),
|
|
2801
|
+
currentPreset.pareto_slider_enabled && React.createElement(
|
|
2802
|
+
"div",
|
|
2803
|
+
{ style: { marginLeft: 22, marginTop: 4, display: "flex", alignItems: "center", gap: 10 } },
|
|
2804
|
+
React.createElement("span", { style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)" } }, t("aggregator.pareto_pos_label") || "Frontier Position:"),
|
|
2805
|
+
React.createElement("input", {
|
|
2806
|
+
type: "range",
|
|
2807
|
+
min: 0,
|
|
2808
|
+
max: 100,
|
|
2809
|
+
step: 5,
|
|
2810
|
+
value: currentPreset.pareto_slider_position ?? 50,
|
|
2811
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, pareto_slider_position: Number(e.target.value) })),
|
|
2812
|
+
style: { cursor: "pointer", flex: 1, maxWidth: 200 }
|
|
2813
|
+
}),
|
|
2814
|
+
React.createElement("span", { style: { fontSize: 12, fontWeight: 600, color: "var(--dsw-alias-label-primary)" } }, String(currentPreset.pareto_slider_position ?? 50) + "%")
|
|
2815
|
+
)
|
|
2816
|
+
),
|
|
2817
|
+
/* Hard Token Budget Guard (#319) */
|
|
2818
|
+
React.createElement(
|
|
2819
|
+
"div",
|
|
2820
|
+
{ style: { display: "flex", flexDirection: "column", gap: 6 } },
|
|
2821
|
+
React.createElement(
|
|
2822
|
+
"label",
|
|
2823
|
+
{ style: { display: "flex", alignItems: "center", gap: 8, fontSize: 13, fontWeight: 600, color: "var(--dsw-alias-label-primary)", cursor: "pointer" } },
|
|
2824
|
+
React.createElement("input", {
|
|
2825
|
+
type: "checkbox",
|
|
2826
|
+
checked: Boolean(currentPreset.hard_budget_guard_enabled),
|
|
2827
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, hard_budget_guard_enabled: e.target.checked })),
|
|
2828
|
+
style: { accentColor: "var(--dsw-alias-state-brand-primary, var(--dsw-alias-label-primary))", cursor: "pointer" }
|
|
2829
|
+
}),
|
|
2830
|
+
t("aggregator.hard_budget_title") || "\u{1F6E1}\uFE0F Hard Token Budget Guard with Auto-Scaling"
|
|
2831
|
+
),
|
|
2832
|
+
React.createElement(
|
|
2833
|
+
"div",
|
|
2834
|
+
{ style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)", marginLeft: 22 } },
|
|
2835
|
+
t("aggregator.hard_budget_desc") || "Pre-calculates turn costs against a hard financial ceiling, automatically scaling down or halting execution."
|
|
2836
|
+
),
|
|
2837
|
+
currentPreset.hard_budget_guard_enabled && React.createElement(
|
|
2838
|
+
"div",
|
|
2839
|
+
{ style: { marginLeft: 22, marginTop: 4, display: "flex", flexWrap: "wrap", alignItems: "center", gap: 12 } },
|
|
2840
|
+
React.createElement(
|
|
2841
|
+
"div",
|
|
2842
|
+
{ style: { display: "flex", alignItems: "center", gap: 6 } },
|
|
2843
|
+
React.createElement("span", { style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)" } }, t("aggregator.hard_budget_limit_label") || "Max Cost (USD):"),
|
|
2844
|
+
React.createElement("input", {
|
|
2845
|
+
type: "number",
|
|
2846
|
+
min: 1e-3,
|
|
2847
|
+
max: 10,
|
|
2848
|
+
step: 5e-3,
|
|
2849
|
+
value: currentPreset.hard_budget_limit_usd ?? 0.02,
|
|
2850
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, hard_budget_limit_usd: Number(e.target.value) })),
|
|
2851
|
+
style: { width: 80, padding: "2px 6px", fontSize: 12, borderRadius: 4, border: "1px solid var(--dsw-alias-border-primary)" }
|
|
2852
|
+
})
|
|
2853
|
+
),
|
|
2854
|
+
React.createElement(
|
|
2855
|
+
"div",
|
|
2856
|
+
{ style: { display: "flex", alignItems: "center", gap: 6 } },
|
|
2857
|
+
React.createElement("span", { style: { fontSize: 12, color: "var(--dsw-alias-label-secondary)" } }, t("aggregator.hard_budget_policy_label") || "Policy:"),
|
|
2858
|
+
React.createElement(
|
|
2859
|
+
"select",
|
|
2860
|
+
{
|
|
2861
|
+
value: currentPreset.hard_budget_policy || "auto_downgrade",
|
|
2862
|
+
onChange: (e) => updateCurrentPreset((p) => ({ ...p, hard_budget_policy: e.target.value })),
|
|
2863
|
+
style: { padding: "2px 6px", fontSize: 12, borderRadius: 4, border: "1px solid var(--dsw-alias-border-primary)" }
|
|
2864
|
+
},
|
|
2865
|
+
React.createElement("option", { value: "auto_downgrade" }, t("aggregator.hard_budget_policy_downgrade") || "Auto-Downgrade"),
|
|
2866
|
+
React.createElement("option", { value: "fail_closed" }, t("aggregator.hard_budget_policy_fail") || "Fail-Closed")
|
|
2867
|
+
)
|
|
2868
|
+
)
|
|
2869
|
+
)
|
|
2870
|
+
),
|
|
2763
2871
|
/* Fallback Judges Chain */
|
|
2764
2872
|
React.createElement(
|
|
2765
2873
|
"div",
|
package/lib/events.js
CHANGED
|
@@ -17,6 +17,8 @@ export const MOA_EVENTS = {
|
|
|
17
17
|
MUTATION_COMPLETE: 'moa/mutation-complete',
|
|
18
18
|
PRIVACY_SANITIZED: 'moa/privacy-sanitized',
|
|
19
19
|
SPECULATIVE_DRAFT: 'moa/speculative-draft',
|
|
20
|
+
PARETO_ADJUSTED: 'moa/pareto-adjusted',
|
|
21
|
+
BUDGET_SCALED: 'moa/budget-scaled',
|
|
20
22
|
CRITIQUE_COMPLETE: 'moa/critique-complete',
|
|
21
23
|
JUDGE_VERDICT: 'moa/judge-verdict',
|
|
22
24
|
CONTEXT_PRUNING: 'moa/context-pruning',
|
package/lib/moa-schema.js
CHANGED
|
@@ -107,6 +107,11 @@ export const PresetSchema = z.object({
|
|
|
107
107
|
redact_ips: z.boolean().default(true),
|
|
108
108
|
speculative_draft_enabled: z.boolean().default(false),
|
|
109
109
|
speculative_block_size: z.number().default(256),
|
|
110
|
+
pareto_slider_enabled: z.boolean().default(false),
|
|
111
|
+
pareto_slider_position: z.number().default(50),
|
|
112
|
+
hard_budget_guard_enabled: z.boolean().default(false),
|
|
113
|
+
hard_budget_limit_usd: z.number().default(0.02),
|
|
114
|
+
hard_budget_policy: z.string().default('auto_downgrade'),
|
|
110
115
|
})
|
|
111
116
|
|
|
112
117
|
export const Config = z.object({
|
|
@@ -160,6 +165,11 @@ export const Config = z.object({
|
|
|
160
165
|
redact_ips: z.boolean().default(true).volatile(),
|
|
161
166
|
speculative_draft_enabled: z.boolean().default(false).volatile(),
|
|
162
167
|
speculative_block_size: z.number().default(256).volatile(),
|
|
168
|
+
pareto_slider_enabled: z.boolean().default(false).volatile(),
|
|
169
|
+
pareto_slider_position: z.number().default(50).volatile(),
|
|
170
|
+
hard_budget_guard_enabled: z.boolean().default(false).volatile(),
|
|
171
|
+
hard_budget_limit_usd: z.number().default(0.02).volatile(),
|
|
172
|
+
hard_budget_policy: z.string().default('auto_downgrade').volatile(),
|
|
163
173
|
smart_routing_model: ModelSlotSchema.default({ provider: '', model: '' }).volatile(),
|
|
164
174
|
prices: z.dict(PriceRow).default({}).volatile(),
|
|
165
175
|
presets: z.array(PresetSchema).default([
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DeepSeek Harness Mixture of Agents (MoA) — Hard Token Budget Guard & Auto-Scaling (#319)
|
|
3
|
+
*
|
|
4
|
+
* Enforces strict financial limits on inference costs. Pre-calculates end-to-end turn
|
|
5
|
+
* expenses (proposers + multi-judge + synthesis) against hard_budget_limit_usd before dispatching.
|
|
6
|
+
*
|
|
7
|
+
* Supports two operational policies when anticipated cost exceeds limit:
|
|
8
|
+
* - auto_downgrade: Automatically substitutes costly cloud slots with zero-cost local edge
|
|
9
|
+
* models or economic tiers, and scales down candidate slots until cost fits under the ceiling.
|
|
10
|
+
* - fail_closed: Halts execution immediately to eliminate unexpected API billing surprises.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { MOA_EVENTS, emitEvent } from '../events.js'
|
|
14
|
+
import { estimateCandidateRunCost, estimateDownstreamCost } from '../moa-budget.js'
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Evaluates projected turn expenses and dynamically scales down ensemble if needed.
|
|
18
|
+
*
|
|
19
|
+
* @param {object} params
|
|
20
|
+
* @param {Array<object>} params.references
|
|
21
|
+
* @param {object} params.aggregator
|
|
22
|
+
* @param {number} params.limitUsd
|
|
23
|
+
* @param {string} [params.policy='auto_downgrade']
|
|
24
|
+
* @param {object} [params.prices={}]
|
|
25
|
+
* @param {number} [params.promptTokens=1500]
|
|
26
|
+
* @param {number} [params.expectedOutputTokens=2048]
|
|
27
|
+
* @param {object} [params.options]
|
|
28
|
+
* @returns {object} Budget evaluation result
|
|
29
|
+
*/
|
|
30
|
+
export function evaluateHardBudgetLimit({
|
|
31
|
+
references = [],
|
|
32
|
+
aggregator = {},
|
|
33
|
+
limitUsd = 0.02,
|
|
34
|
+
policy = 'auto_downgrade',
|
|
35
|
+
prices = {},
|
|
36
|
+
promptTokens = 1500,
|
|
37
|
+
expectedOutputTokens = 2048,
|
|
38
|
+
options = {},
|
|
39
|
+
}) {
|
|
40
|
+
const { ctx, runId, onProgress } = options
|
|
41
|
+
|
|
42
|
+
// Calculate baseline projected cost
|
|
43
|
+
const candCost = estimateCandidateRunCost(references, prices, promptTokens, expectedOutputTokens)
|
|
44
|
+
const downstreamCost = estimateDownstreamCost(
|
|
45
|
+
aggregator,
|
|
46
|
+
[],
|
|
47
|
+
references.length,
|
|
48
|
+
prices,
|
|
49
|
+
expectedOutputTokens,
|
|
50
|
+
promptTokens
|
|
51
|
+
)
|
|
52
|
+
const initialProjectedCost = Number((candCost.totalEstimatedUsd + downstreamCost).toFixed(5))
|
|
53
|
+
|
|
54
|
+
if (initialProjectedCost <= limitUsd) {
|
|
55
|
+
return {
|
|
56
|
+
ok: true,
|
|
57
|
+
scaled: false,
|
|
58
|
+
references,
|
|
59
|
+
aggregator,
|
|
60
|
+
projectedCostUsd: initialProjectedCost,
|
|
61
|
+
initialProjectedCost,
|
|
62
|
+
limitUsd,
|
|
63
|
+
warning: null,
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Cost exceeds limit
|
|
68
|
+
if (policy === 'fail_closed') {
|
|
69
|
+
const errorMsg = `[BudgetGuard] Projected turn cost ($${initialProjectedCost}) exceeds hard limit ($${limitUsd}). Execution halted.`
|
|
70
|
+
if (typeof onProgress === 'function') {
|
|
71
|
+
onProgress(`🛑 *${errorMsg}*\n`)
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
ok: false,
|
|
75
|
+
scaled: false,
|
|
76
|
+
error: errorMsg,
|
|
77
|
+
projectedCostUsd: initialProjectedCost,
|
|
78
|
+
initialProjectedCost,
|
|
79
|
+
limitUsd,
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// policy: auto_downgrade
|
|
84
|
+
let workingRefs = [...references]
|
|
85
|
+
let workingAggregator = { ...aggregator }
|
|
86
|
+
let scaled = false
|
|
87
|
+
|
|
88
|
+
// Step 1: Replace expensive cloud proposers with local edge models (zero cost)
|
|
89
|
+
for (let i = workingRefs.length - 1; i >= 0; i--) {
|
|
90
|
+
const currentSlot = workingRefs[i]
|
|
91
|
+
if (currentSlot.provider !== 'ollama' && currentSlot.provider !== 'local') {
|
|
92
|
+
workingRefs[i] = {
|
|
93
|
+
...currentSlot,
|
|
94
|
+
provider: 'ollama',
|
|
95
|
+
model: 'qwen2.5-coder:7b',
|
|
96
|
+
role_persona: currentSlot.role_persona || 'proposer',
|
|
97
|
+
}
|
|
98
|
+
scaled = true
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const testCandCost = estimateCandidateRunCost(workingRefs, prices, promptTokens, expectedOutputTokens)
|
|
102
|
+
const testDownstream = estimateDownstreamCost(workingAggregator, [], workingRefs.length, prices, expectedOutputTokens, promptTokens)
|
|
103
|
+
if ((testCandCost.totalEstimatedUsd + testDownstream) <= limitUsd) {
|
|
104
|
+
break
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Step 2: If still exceeding, replace expensive aggregator with economical model or local
|
|
109
|
+
let currentCost = Number((estimateCandidateRunCost(workingRefs, prices, promptTokens, expectedOutputTokens).totalEstimatedUsd +
|
|
110
|
+
estimateDownstreamCost(workingAggregator, [], workingRefs.length, prices, expectedOutputTokens, promptTokens)).toFixed(5))
|
|
111
|
+
|
|
112
|
+
if (currentCost > limitUsd && workingAggregator.provider !== 'ollama') {
|
|
113
|
+
workingAggregator = { provider: 'deepseek', model: 'deepseek-chat' }
|
|
114
|
+
scaled = true
|
|
115
|
+
currentCost = Number((estimateCandidateRunCost(workingRefs, prices, promptTokens, expectedOutputTokens).totalEstimatedUsd +
|
|
116
|
+
estimateDownstreamCost(workingAggregator, [], workingRefs.length, prices, expectedOutputTokens, promptTokens)).toFixed(5))
|
|
117
|
+
|
|
118
|
+
if (currentCost > limitUsd) {
|
|
119
|
+
workingAggregator = { provider: 'ollama', model: 'qwen2.5-coder:7b' }
|
|
120
|
+
currentCost = 0.0
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Step 3: If still exceeding, prune candidates down to 1
|
|
125
|
+
if (currentCost > limitUsd && workingRefs.length > 1) {
|
|
126
|
+
workingRefs = [workingRefs[0]]
|
|
127
|
+
scaled = true
|
|
128
|
+
currentCost = Number((estimateCandidateRunCost(workingRefs, prices, promptTokens, expectedOutputTokens).totalEstimatedUsd +
|
|
129
|
+
estimateDownstreamCost(workingAggregator, [], workingRefs.length, prices, expectedOutputTokens, promptTokens)).toFixed(5))
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (scaled) {
|
|
133
|
+
if (typeof onProgress === 'function') {
|
|
134
|
+
onProgress(`🛡️ *Budget Guard Auto-Scaling*: Scaled ensemble from $${initialProjectedCost} to $${currentCost} to fit under $${limitUsd} ceiling.\n`)
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (ctx) {
|
|
138
|
+
emitEvent(ctx, MOA_EVENTS.BUDGET_SCALED, {
|
|
139
|
+
runId,
|
|
140
|
+
originalCostUsd: initialProjectedCost,
|
|
141
|
+
scaledCostUsd: currentCost,
|
|
142
|
+
limitUsd,
|
|
143
|
+
policy,
|
|
144
|
+
})
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
return {
|
|
149
|
+
ok: true,
|
|
150
|
+
scaled,
|
|
151
|
+
references: workingRefs,
|
|
152
|
+
aggregator: workingAggregator,
|
|
153
|
+
projectedCostUsd: currentCost,
|
|
154
|
+
initialProjectedCost,
|
|
155
|
+
limitUsd,
|
|
156
|
+
warning: scaled ? `Budget Guard Auto-Scaled: initial $${initialProjectedCost} $\\to$ $${currentCost} (ceiling $${limitUsd})` : null,
|
|
157
|
+
}
|
|
158
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DeepSeek Harness Mixture of Agents (MoA) — Cost/Quality Pareto Frontier Slider (#307)
|
|
3
|
+
*
|
|
4
|
+
* Provides reactive calculation of the optimal ensemble trade-off between Cost ($),
|
|
5
|
+
* Quality (Rigorous Consensus), and Latency (seconds) along the Pareto Frontier.
|
|
6
|
+
* Dynamically maps a single slider value (0–100%) to curated model configurations:
|
|
7
|
+
* - 0–25%: Local Free Edge Tier ($0.00 cost, ultra-low latency)
|
|
8
|
+
* - 26–50%: Economic Tier (Low-cost cloud models + deepseek-chat)
|
|
9
|
+
* - 51–75%: Balanced Production Tier (Diverse candidates + deepseek-reasoner / Sonnet)
|
|
10
|
+
* - 76–100%: Ultra-Rigorous Consensus Tier (Flagship ensemble + Multi-Judge panel)
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { MOA_EVENTS, emitEvent } from '../events.js'
|
|
14
|
+
import { estimateCandidateRunCost, estimateDownstreamCost } from '../moa-budget.js'
|
|
15
|
+
|
|
16
|
+
export const PARETO_TIERS = [
|
|
17
|
+
{
|
|
18
|
+
maxPos: 25,
|
|
19
|
+
name: 'local_free',
|
|
20
|
+
label: '⚡ 100% Free / Local Edge',
|
|
21
|
+
description: 'Zero API costs. Local models on MiniPC (Qwen 2.5 Coder / Ollama) for maximum speed and privacy.',
|
|
22
|
+
defaultReferences: [
|
|
23
|
+
{ provider: 'ollama', model: 'qwen2.5-coder:7b', role_persona: 'proposer' },
|
|
24
|
+
{ provider: 'ollama', model: 'deepseek-r1:8b', role_persona: 'challenger' },
|
|
25
|
+
],
|
|
26
|
+
defaultAggregator: { provider: 'ollama', model: 'qwen2.5-coder:7b' },
|
|
27
|
+
estimatedCostUsd: 0.0,
|
|
28
|
+
estimatedLatencySec: 4,
|
|
29
|
+
multiJudge: false,
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
maxPos: 50,
|
|
33
|
+
name: 'economic',
|
|
34
|
+
label: '💡 Economic Tier',
|
|
35
|
+
description: 'Low-cost fast models. Local edge proposers verified by an economical cloud aggregator.',
|
|
36
|
+
defaultReferences: [
|
|
37
|
+
{ provider: 'ollama', model: 'qwen2.5-coder:7b', role_persona: 'proposer' },
|
|
38
|
+
{ provider: 'deepseek', model: 'deepseek-chat', role_persona: 'challenger' },
|
|
39
|
+
],
|
|
40
|
+
defaultAggregator: { provider: 'deepseek', model: 'deepseek-chat' },
|
|
41
|
+
estimatedCostUsd: 0.002,
|
|
42
|
+
estimatedLatencySec: 7,
|
|
43
|
+
multiJudge: false,
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
maxPos: 75,
|
|
47
|
+
name: 'balanced',
|
|
48
|
+
label: '⚖️ Optimal Balanced Tier',
|
|
49
|
+
description: 'High hypothesis diversity with reasoning models and flagship aggregator.',
|
|
50
|
+
defaultReferences: [
|
|
51
|
+
{ provider: 'deepseek', model: 'deepseek-chat', role_persona: 'proposer' },
|
|
52
|
+
{ provider: 'deepseek', model: 'deepseek-reasoner', role_persona: 'challenger' },
|
|
53
|
+
{ provider: 'ollama', model: 'qwen2.5-coder:7b', role_persona: 'minimalist' },
|
|
54
|
+
],
|
|
55
|
+
defaultAggregator: { provider: 'deepseek', model: 'deepseek-reasoner' },
|
|
56
|
+
estimatedCostUsd: 0.009,
|
|
57
|
+
estimatedLatencySec: 12,
|
|
58
|
+
multiJudge: false,
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
maxPos: 100,
|
|
62
|
+
name: 'ultra_rigorous',
|
|
63
|
+
label: '🏆 Ultra-Rigorous Consensus',
|
|
64
|
+
description: 'Maximum accuracy. Flagship reasoning models with multi-judge panel and adversarial peer critique.',
|
|
65
|
+
defaultReferences: [
|
|
66
|
+
{ provider: 'deepseek', model: 'deepseek-reasoner', role_persona: 'proposer' },
|
|
67
|
+
{ provider: 'anthropic', model: 'claude-3-5-sonnet', role_persona: 'challenger' },
|
|
68
|
+
{ provider: 'openai', model: 'gpt-4o', role_persona: 'adversary' },
|
|
69
|
+
],
|
|
70
|
+
defaultAggregator: { provider: 'deepseek', model: 'deepseek-reasoner' },
|
|
71
|
+
estimatedCostUsd: 0.038,
|
|
72
|
+
estimatedLatencySec: 22,
|
|
73
|
+
multiJudge: true,
|
|
74
|
+
},
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Returns the Pareto tier definition for a given slider position (0-100).
|
|
79
|
+
*
|
|
80
|
+
* @param {number} position
|
|
81
|
+
* @returns {object} Tier configuration
|
|
82
|
+
*/
|
|
83
|
+
export function getParetoProfile(position = 50) {
|
|
84
|
+
const clamped = Math.max(0, Math.min(100, Number(position) || 0))
|
|
85
|
+
for (const tier of PARETO_TIERS) {
|
|
86
|
+
if (clamped <= tier.maxPos) {
|
|
87
|
+
return { ...tier, position: clamped }
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return { ...PARETO_TIERS[PARETO_TIERS.length - 1], position: clamped }
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Calculates dynamically predicted cost and latency for an ensemble along the Pareto Frontier.
|
|
95
|
+
*
|
|
96
|
+
* @param {number} sliderPos
|
|
97
|
+
* @param {Array<object>} [availableModels]
|
|
98
|
+
* @param {object} [options]
|
|
99
|
+
* @returns {object} Ensemble recommendation
|
|
100
|
+
*/
|
|
101
|
+
export function calculateParetoEnsemble(sliderPos = 50, availableModels = [], options = {}) {
|
|
102
|
+
const profile = getParetoProfile(sliderPos)
|
|
103
|
+
const { prices = {}, promptTokens = 1500, expectedOutputTokens = 2048 } = options
|
|
104
|
+
|
|
105
|
+
// Calculate upfront cost
|
|
106
|
+
const candCost = estimateCandidateRunCost(profile.defaultReferences, prices, promptTokens, expectedOutputTokens)
|
|
107
|
+
const downstreamCost = estimateDownstreamCost(
|
|
108
|
+
profile.defaultAggregator,
|
|
109
|
+
profile.multiJudge ? profile.defaultReferences : [],
|
|
110
|
+
profile.defaultReferences.length,
|
|
111
|
+
prices,
|
|
112
|
+
expectedOutputTokens,
|
|
113
|
+
promptTokens
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
const calculatedCostUsd = profile.name === 'local_free'
|
|
117
|
+
? 0.0
|
|
118
|
+
: Number((candCost.totalEstimatedUsd + downstreamCost).toFixed(5))
|
|
119
|
+
|
|
120
|
+
return {
|
|
121
|
+
tier: profile.name,
|
|
122
|
+
label: profile.label,
|
|
123
|
+
description: profile.description,
|
|
124
|
+
sliderPosition: profile.position,
|
|
125
|
+
estimatedCostUsd: calculatedCostUsd > 0 ? calculatedCostUsd : profile.estimatedCostUsd,
|
|
126
|
+
estimatedLatencySec: profile.estimatedLatencySec,
|
|
127
|
+
recommendedReferences: profile.defaultReferences,
|
|
128
|
+
recommendedAggregator: profile.defaultAggregator,
|
|
129
|
+
multiJudgeEnabled: profile.multiJudge,
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Adapts preset configuration based on Pareto slider position if enabled.
|
|
135
|
+
*
|
|
136
|
+
* @param {object} preset
|
|
137
|
+
* @param {object} [options]
|
|
138
|
+
* @returns {object} Adjusted preset
|
|
139
|
+
*/
|
|
140
|
+
export function applyParetoAdjustment(preset, options = {}) {
|
|
141
|
+
if (!preset?.pareto_slider_enabled) return preset
|
|
142
|
+
|
|
143
|
+
const { ctx, runId, onProgress } = options
|
|
144
|
+
const position = preset.pareto_slider_position ?? 50
|
|
145
|
+
const recommendation = calculateParetoEnsemble(position, options.availableModels || [], options)
|
|
146
|
+
|
|
147
|
+
if (typeof onProgress === 'function') {
|
|
148
|
+
onProgress(`📈 *Pareto Frontier*: Active profile ${recommendation.label} (~$${recommendation.estimatedCostUsd}, ~${recommendation.estimatedLatencySec}s)\n`)
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (ctx) {
|
|
152
|
+
emitEvent(ctx, MOA_EVENTS.PARETO_ADJUSTED, {
|
|
153
|
+
runId,
|
|
154
|
+
position,
|
|
155
|
+
tier: recommendation.tier,
|
|
156
|
+
estimatedCostUsd: recommendation.estimatedCostUsd,
|
|
157
|
+
estimatedLatencySec: recommendation.estimatedLatencySec,
|
|
158
|
+
})
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return {
|
|
162
|
+
...preset,
|
|
163
|
+
reference_models: recommendation.recommendedReferences,
|
|
164
|
+
aggregator: recommendation.recommendedAggregator,
|
|
165
|
+
multi_judge_enabled: recommendation.multiJudgeEnabled || preset.multi_judge_enabled,
|
|
166
|
+
}
|
|
167
|
+
}
|
package/lib/pipeline/runner.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { applyParetoAdjustment } from './pareto-frontier.js'
|
|
2
|
+
import { evaluateHardBudgetLimit } from './budget-guard.js'
|
|
1
3
|
import { sanitizeCandidatesForCloudJudge } from './privacy-sanitizer.js'
|
|
2
4
|
import { evaluateJudgeEscalation } from './escalation.js'
|
|
3
5
|
import { calculateSelfConsistencyAgreement } from './self-consistency.js'
|
|
@@ -50,9 +52,36 @@ export async function runMoAPipeline({
|
|
|
50
52
|
timestamp: new Date(startTime).toISOString(),
|
|
51
53
|
})
|
|
52
54
|
|
|
55
|
+
// Pre-Stage 0.5: Pareto Frontier Slider & Hard Budget Guard (#307, #319)
|
|
56
|
+
let effectivePreset = preset
|
|
57
|
+
if (effectivePreset?.pareto_slider_enabled) {
|
|
58
|
+
effectivePreset = applyParetoAdjustment(effectivePreset, { ctx, runId, onProgress, prices })
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if (effectivePreset?.hard_budget_guard_enabled) {
|
|
62
|
+
const budgetRes = evaluateHardBudgetLimit({
|
|
63
|
+
references: effectivePreset.reference_models || [],
|
|
64
|
+
aggregator: effectivePreset.aggregator || {},
|
|
65
|
+
limitUsd: effectivePreset.hard_budget_limit_usd ?? 0.02,
|
|
66
|
+
policy: effectivePreset.hard_budget_policy || 'auto_downgrade',
|
|
67
|
+
prices,
|
|
68
|
+
options: { ctx, runId, onProgress },
|
|
69
|
+
})
|
|
70
|
+
if (!budgetRes.ok) {
|
|
71
|
+
return { error: new Error(budgetRes.error) }
|
|
72
|
+
}
|
|
73
|
+
if (budgetRes.scaled) {
|
|
74
|
+
effectivePreset = {
|
|
75
|
+
...effectivePreset,
|
|
76
|
+
reference_models: budgetRes.references,
|
|
77
|
+
aggregator: budgetRes.aggregator,
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
53
82
|
// Stage 1: Context & Configuration
|
|
54
83
|
const ctxStage = await stageProjectContext({
|
|
55
|
-
userPrompt, messages, preset, cwd, prices, maxTokens:
|
|
84
|
+
userPrompt, messages, preset: effectivePreset, cwd, prices, maxTokens: effectivePreset?.max_tokens, startTime, onProgress, ctx,
|
|
56
85
|
})
|
|
57
86
|
if (!ctxStage.ok) return ctxStage.abortPayload
|
|
58
87
|
|
|
@@ -307,7 +336,6 @@ export async function runMoAPipeline({
|
|
|
307
336
|
let effectivePrimaryJudge = ctxStage.primaryJudge
|
|
308
337
|
let effectiveFallbackJudges = ctxStage.fallbackJudges
|
|
309
338
|
let effectiveJudgesChain = ctxStage.judgesChain
|
|
310
|
-
let effectivePreset = preset
|
|
311
339
|
|
|
312
340
|
if (escalationResult.escalated) {
|
|
313
341
|
emitEvent(ctx, MOA_EVENTS.JUDGE_ESCALATION, {
|
package/lib/pricing.js
CHANGED
|
@@ -152,6 +152,11 @@ export function resolveModelRates(slot = {}, customPrices = {}, cachePath = getD
|
|
|
152
152
|
const fullKey = provider ? `${provider}/${model}` : model
|
|
153
153
|
const colonKey = provider ? `${provider}:${model}` : model
|
|
154
154
|
|
|
155
|
+
// 0. Free local edge providers
|
|
156
|
+
if (provider === 'ollama' || provider === 'local') {
|
|
157
|
+
return { input: 0, output: 0, cacheHit: 0 }
|
|
158
|
+
}
|
|
159
|
+
|
|
155
160
|
// 1. Check custom user overrides in config
|
|
156
161
|
if (customPrices && typeof customPrices === 'object') {
|
|
157
162
|
if (customPrices[fullKey]) return normalizeRates(customPrices[fullKey])
|
|
@@ -278,9 +283,9 @@ export function resolveModelRates(slot = {}, customPrices = {}, cachePath = getD
|
|
|
278
283
|
function normalizeRates(rate) {
|
|
279
284
|
if (!rate || typeof rate !== 'object') return { ...FALLBACK_RATES }
|
|
280
285
|
return {
|
|
281
|
-
input: Number(rate.input ?? rate.prompt ?? FALLBACK_RATES.input),
|
|
282
|
-
output: Number(rate.output ?? rate.completion ?? FALLBACK_RATES.output),
|
|
283
|
-
cacheHit: Number(rate.cacheHit ?? rate.cache_hit ?? FALLBACK_RATES.cacheHit),
|
|
286
|
+
input: Number(rate.input ?? rate.prompt ?? rate.prompt_per_million ?? FALLBACK_RATES.input),
|
|
287
|
+
output: Number(rate.output ?? rate.completion ?? rate.completion_per_million ?? FALLBACK_RATES.output),
|
|
288
|
+
cacheHit: Number(rate.cacheHit ?? rate.cache_hit ?? rate.cache_hit_per_million ?? FALLBACK_RATES.cacheHit),
|
|
284
289
|
}
|
|
285
290
|
}
|
|
286
291
|
|