adaptive-memory-multi-model-router 2.14.40 → 2.14.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ensemble/multiRoundDialog.d.ts +113 -0
- package/dist/ensemble/multiRoundDialog.js +284 -0
- package/dist/ensemble/multiRoundDialog.js.map +1 -0
- package/dist/ensemble/shapleyValue.d.ts +119 -0
- package/dist/ensemble/shapleyValue.js +280 -0
- package/dist/ensemble/shapleyValue.js.map +1 -0
- package/dist/ensemble.d.ts +18 -3
- package/dist/ensemble.js +90 -16
- package/dist/index.d.ts +3 -0
- package/dist/index.js +28 -1
- package/dist/integrations/scienceAdapter.d.ts +77 -0
- package/dist/integrations/scienceAdapter.js +8 -7
- package/dist/integrations/scienceAdapter.js.map +1 -0
- package/dist/tui/dashboard.js +1 -0
- package/dist/tui/dashboard.js.map +1 -1
- package/package.json +1 -1
- package/src/ensemble/multiRoundDialog.ts +374 -0
- package/src/ensemble/shapleyValue.ts +353 -0
- package/src/ensemble.ts +117 -26
- package/src/index.ts +24 -0
- package/src/integrations/scienceAdapter.ts +2 -2
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Enhanced Shapley Value Calculator for Ensemble Credit Assignment
|
|
3
|
+
*
|
|
4
|
+
* Incorporates game theory concepts for efficient and fair credit distribution:
|
|
5
|
+
*
|
|
6
|
+
* 1. ETHNOCENTRISM (In-group Loyalty Adjustment):
|
|
7
|
+
* - Players (models) have historical loyalty biases toward certain partners
|
|
8
|
+
* - Models that collaborate successfully develop "trust bonds"
|
|
9
|
+
* - Loyalty increases marginal contribution of trusted partners
|
|
10
|
+
* - Math: L[i,j] = exponential moving avg of historical success(i with j)
|
|
11
|
+
*
|
|
12
|
+
* 2. HANDICAP PRINCIPLE (Zahavi, 1975 - Costly Signaling):
|
|
13
|
+
* - Honest signals require costly investment
|
|
14
|
+
* - Models providing correct answers despite cost signal reliability
|
|
15
|
+
* - Math: H[i] = cost_i * reliability_i (handicap bonus)
|
|
16
|
+
*
|
|
17
|
+
* 3. CORE SHAPLEY VALUE:
|
|
18
|
+
* - φ_i = Σ_{S⊆N\{i}} (|S|! * (n-|S|-1)! / n!) * (v(S∪{i}) - v(S))
|
|
19
|
+
*
|
|
20
|
+
* Combined: φ_i* = α*Shapley_i + β*Loyalty_i + γ*Handicap_i
|
|
21
|
+
* Where α + β + γ = 1 (normalized weights)
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
export interface ShapleyConfig {
|
|
25
|
+
nPermutations?: number;
|
|
26
|
+
useMonteCarlo?: boolean;
|
|
27
|
+
alpha?: number; // Shapley weight (default 0.5)
|
|
28
|
+
beta?: number; // Loyalty/ethnocentrism weight (default 0.3)
|
|
29
|
+
gamma?: number; // Handicap weight (default 0.2)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface ModelContribution {
|
|
33
|
+
modelId: string;
|
|
34
|
+
shapleyValue: number;
|
|
35
|
+
loyaltyValue: number;
|
|
36
|
+
handicapValue: number;
|
|
37
|
+
combinedCredit: number;
|
|
38
|
+
marginalContributions: number[];
|
|
39
|
+
timesSelected: number;
|
|
40
|
+
averageMarginal: number;
|
|
41
|
+
reliabilityScore: number;
|
|
42
|
+
costInvested: number;
|
|
43
|
+
ethnocentrismBias: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
type AccuracyFunction = (modelIds: string[]) => number;
|
|
47
|
+
|
|
48
|
+
// ============ ETHNOCENTRISM: LOYALTY MATRIX ============
|
|
49
|
+
|
|
50
|
+
export interface LoyaltyConfig {
|
|
51
|
+
decayRate?: number; // EMA decay (0-1, higher = more recent)
|
|
52
|
+
minInteractions?: number; // Min history before loyalty applies
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const DEFAULT_LOYALTY: LoyaltyConfig = { decayRate: 0.9, minInteractions: 3 };
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Loyalty Matrix L[i][j] = loyalty of model i toward model j (0-1)
|
|
59
|
+
* Models develop trust through successful collaborations
|
|
60
|
+
*/
|
|
61
|
+
export class LoyaltyMatrix {
|
|
62
|
+
private matrix = new Map<string, Map<string, number>>();
|
|
63
|
+
private counts = new Map<string, Map<string, number>>();
|
|
64
|
+
private config: Required<LoyaltyConfig>;
|
|
65
|
+
|
|
66
|
+
constructor(cfg: LoyaltyConfig = {}) {
|
|
67
|
+
this.config = { ...DEFAULT_LOYALTY, ...cfg } as Required<LoyaltyConfig>;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Record successful collaboration between two models */
|
|
71
|
+
recordSuccess(i: string, j: string, weight = 1): void {
|
|
72
|
+
if (!this.matrix.has(i)) { this.matrix.set(i, new Map()); this.counts.set(i, new Map()); }
|
|
73
|
+
if (!this.matrix.get(i)!.has(j)) { this.matrix.get(i)!.set(j, 0); this.counts.get(i)!.set(j, 0); }
|
|
74
|
+
const prev = this.matrix.get(i)!.get(j)!;
|
|
75
|
+
const cnt = this.counts.get(i)!.get(j)!;
|
|
76
|
+
const ema = this.config.decayRate * prev + (1 - this.config.decayRate) * weight;
|
|
77
|
+
this.matrix.get(i)!.set(j, ema);
|
|
78
|
+
this.counts.get(i)!.set(j, cnt + 1);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Get loyalty of model i toward model j */
|
|
82
|
+
getLoyalty(i: string, j: string): number {
|
|
83
|
+
if (!this.matrix.has(i) || !this.matrix.get(i)!.has(j)) return 0;
|
|
84
|
+
if (this.counts.get(i)!.get(j)! < this.config.minInteractions) return 0;
|
|
85
|
+
return this.matrix.get(i)!.get(j)!;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Ethnocentrism = average loyalty toward all partners */
|
|
89
|
+
ethnoCentrism(model: string, allModels: string[]): number {
|
|
90
|
+
const loyalties = allModels.map(m => this.getLoyalty(model, m));
|
|
91
|
+
const avg = loyalties.reduce((a, b) => a + b, 0) / Math.max(1, allModels.length);
|
|
92
|
+
return Math.min(1, avg);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// ============ HANDICAP PRINCIPLE: COSTLY SIGNALING ============
|
|
97
|
+
|
|
98
|
+
export interface HandicapConfig {
|
|
99
|
+
costSensitivity?: number; // How much cost affects bonus
|
|
100
|
+
reliabilityWeight?: number; // How much reliability affects bonus
|
|
101
|
+
minCostThreshold?: number; // Min cost for handicap signal
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const DEFAULT_HANDICAP: HandicapConfig = { costSensitivity: 0.5, reliabilityWeight: 0.5, minCostThreshold: 0.0001 };
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Handicap Principle (Zahavi, 1975):
|
|
108
|
+
* "A handicapping signal is honest because it is costly to produce"
|
|
109
|
+
*
|
|
110
|
+
* Models spending more tokens on answers despite being correct signal reliability.
|
|
111
|
+
* H[i] = cost_i * reliability_i (higher = more reliable despite cost)
|
|
112
|
+
*/
|
|
113
|
+
export class HandicapCalculator {
|
|
114
|
+
private costs = new Map<string, number>();
|
|
115
|
+
private correct = new Map<string, number>();
|
|
116
|
+
private totals = new Map<string, number>();
|
|
117
|
+
private config: Required<HandicapConfig>;
|
|
118
|
+
|
|
119
|
+
constructor(cfg: HandicapConfig = {}) {
|
|
120
|
+
this.config = { ...DEFAULT_HANDICAP, ...cfg } as Required<HandicapConfig>;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Record performance: cost spent and whether answer was correct */
|
|
124
|
+
record(model: string, cost: number, isCorrect: boolean): void {
|
|
125
|
+
this.costs.set(model, (this.costs.get(model) || 0) + cost);
|
|
126
|
+
this.totals.set(model, (this.totals.get(model) || 0) + 1);
|
|
127
|
+
if (isCorrect) this.correct.set(model, (this.correct.get(model) || 0) + 1);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Reliability = correct / total (prior probability of being right) */
|
|
131
|
+
reliability(model: string): number {
|
|
132
|
+
const total = this.totals.get(model) || 0;
|
|
133
|
+
if (total === 0) return 0.5; // Unknown = neutral
|
|
134
|
+
return (this.correct.get(model) || 0) / total;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Average cost invested by model */
|
|
138
|
+
avgCost(model: string): number {
|
|
139
|
+
const total = this.totals.get(model) || 0;
|
|
140
|
+
if (total === 0) return 0;
|
|
141
|
+
return (this.costs.get(model) || 0) / total;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Handicap bonus = honest signal of quality
|
|
146
|
+
* H[i] = (cost_i - minCost) / maxCost * reliability_i
|
|
147
|
+
* Higher cost investment + higher reliability = higher handicap
|
|
148
|
+
*/
|
|
149
|
+
handicap(model: string, maxCost = 0.01): number {
|
|
150
|
+
const cost = this.avgCost(model);
|
|
151
|
+
if (cost < this.config.minCostThreshold) return 0;
|
|
152
|
+
const rel = this.reliability(model);
|
|
153
|
+
const costNorm = Math.min(1, cost / maxCost);
|
|
154
|
+
// Honest signal: cost * reliability (costly and correct = reliable)
|
|
155
|
+
return this.config.costSensitivity * costNorm * rel +
|
|
156
|
+
this.config.reliabilityWeight * rel;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ============ CORE SHAPLEY VALUE ============
|
|
161
|
+
|
|
162
|
+
function factorial(n: number): number {
|
|
163
|
+
if (n <= 1) return 1;
|
|
164
|
+
return n * factorial(n - 1);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Exact Shapley calculation (factorial enumeration)
|
|
169
|
+
* For n <= 6 models
|
|
170
|
+
*/
|
|
171
|
+
function exactShapley(
|
|
172
|
+
modelIds: string[],
|
|
173
|
+
accuracyFn: AccuracyFunction
|
|
174
|
+
): Map<string, number> {
|
|
175
|
+
const n = modelIds.length;
|
|
176
|
+
const shapley = new Map<string, number>();
|
|
177
|
+
modelIds.forEach(m => shapley.set(m, 0));
|
|
178
|
+
|
|
179
|
+
// All permutations via Heap's algorithm
|
|
180
|
+
function permute(arr: string[], callback: (p: string[]) => void): void {
|
|
181
|
+
function generate(idx: number): void {
|
|
182
|
+
if (idx === arr.length) { callback([...arr]); return; }
|
|
183
|
+
for (let i = idx; i < arr.length; i++) {
|
|
184
|
+
[arr[idx], arr[i]] = [arr[i], arr[idx]];
|
|
185
|
+
generate(idx + 1);
|
|
186
|
+
[arr[idx], arr[i]] = [arr[i], arr[idx]];
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
generate(0);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const permutations: string[][] = [];
|
|
193
|
+
const work = [...modelIds];
|
|
194
|
+
permute(work, p => permutations.push(p));
|
|
195
|
+
|
|
196
|
+
for (const perm of permutations) {
|
|
197
|
+
let acc = 0;
|
|
198
|
+
const subset: string[] = [];
|
|
199
|
+
for (let i = 0; i < perm.length; i++) {
|
|
200
|
+
const model = perm[i];
|
|
201
|
+
const newAcc = accuracyFn([...subset, model]);
|
|
202
|
+
const marginal = newAcc - acc;
|
|
203
|
+
const weight = factorial(subset.length) * factorial(n - subset.length - 1) / factorial(n);
|
|
204
|
+
shapley.set(model, shapley.get(model)! + weight * marginal);
|
|
205
|
+
subset.push(model);
|
|
206
|
+
acc = newAcc;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return shapley;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Monte Carlo approximation for n > 6 models
|
|
215
|
+
*/
|
|
216
|
+
function monteCarloShapley(
|
|
217
|
+
modelIds: string[],
|
|
218
|
+
accuracyFn: AccuracyFunction,
|
|
219
|
+
nPerms: number
|
|
220
|
+
): Map<string, number> {
|
|
221
|
+
const n = modelIds.length;
|
|
222
|
+
const shapley = new Map<string, number>();
|
|
223
|
+
modelIds.forEach(m => shapley.set(m, 0));
|
|
224
|
+
|
|
225
|
+
for (let iter = 0; iter < nPerms; iter++) {
|
|
226
|
+
const perm = [...modelIds].sort(() => Math.random() - 0.5);
|
|
227
|
+
let acc = 0;
|
|
228
|
+
const subset: string[] = [];
|
|
229
|
+
for (const model of perm) {
|
|
230
|
+
const newAcc = accuracyFn([...subset, model]);
|
|
231
|
+
const marginal = newAcc - acc;
|
|
232
|
+
shapley.set(model, shapley.get(model)! + marginal);
|
|
233
|
+
subset.push(model);
|
|
234
|
+
acc = newAcc;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// Average
|
|
239
|
+
for (const [m, v] of shapley) shapley.set(m, v / nPerms);
|
|
240
|
+
return shapley;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Combined enhanced Shapley value with ethnocentrism and handicap
|
|
245
|
+
* φ_i* = α*φ_i(Shapley) + β*ε_i(Ethnocentrism) + γ*H_i(Handicap)
|
|
246
|
+
*/
|
|
247
|
+
export function calculateEnhancedShapley(
|
|
248
|
+
modelIds: string[],
|
|
249
|
+
accuracyFn: AccuracyFunction,
|
|
250
|
+
loyalty: LoyaltyMatrix,
|
|
251
|
+
handicap: HandicapCalculator,
|
|
252
|
+
cfg: ShapleyConfig = {}
|
|
253
|
+
): Map<string, ModelContribution> {
|
|
254
|
+
const { alpha = 0.5, beta = 0.3, gamma = 0.2, nPermutations = 1000, useMonteCarlo = false } = cfg;
|
|
255
|
+
const results = new Map<string, ModelContribution>();
|
|
256
|
+
|
|
257
|
+
// 1. Core Shapley values
|
|
258
|
+
const shapleyValues = (modelIds.length <= 6 && !useMonteCarlo)
|
|
259
|
+
? exactShapley(modelIds, accuracyFn)
|
|
260
|
+
: monteCarloShapley(modelIds, accuracyFn, nPermutations);
|
|
261
|
+
|
|
262
|
+
// 2. Normalize Shapley to [0,1]
|
|
263
|
+
const shapSum = [...shapleyValues.values()].reduce((a, b) => a + b, 0);
|
|
264
|
+
const maxShap = Math.max(...shapleyValues.values(), 0.001);
|
|
265
|
+
|
|
266
|
+
// 3. Compute ethnocentrism and handicap for each model
|
|
267
|
+
for (const model of modelIds) {
|
|
268
|
+
const shap = shapleyValues.get(model)!;
|
|
269
|
+
const ethn = loyalty.ethnoCentrism(model, modelIds);
|
|
270
|
+
const hand = handicap.handicap(model);
|
|
271
|
+
|
|
272
|
+
// Normalized values
|
|
273
|
+
const normShap = shap / maxShap;
|
|
274
|
+
const combined = alpha * normShap + beta * ethn + gamma * hand;
|
|
275
|
+
|
|
276
|
+
results.set(model, {
|
|
277
|
+
modelId: model,
|
|
278
|
+
shapleyValue: shap,
|
|
279
|
+
loyaltyValue: ethn,
|
|
280
|
+
handicapValue: hand,
|
|
281
|
+
combinedCredit: combined,
|
|
282
|
+
marginalContributions: [],
|
|
283
|
+
timesSelected: 0,
|
|
284
|
+
averageMarginal: shap / Math.max(1, modelIds.length),
|
|
285
|
+
reliabilityScore: handicap.reliability(model),
|
|
286
|
+
costInvested: handicap.avgCost(model),
|
|
287
|
+
ethnocentrismBias: ethn,
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// 4. Normalize combined credits to sum to 1
|
|
292
|
+
const totalCredit = [...results.values()].reduce((s, r) => s + r.combinedCredit, 0);
|
|
293
|
+
if (totalCredit > 0) {
|
|
294
|
+
for (const [, r] of results) r.combinedCredit /= totalCredit;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
return results;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** Create ensemble accuracy function for Shapley calculation */
|
|
301
|
+
export function createAccuracyFn(
|
|
302
|
+
groundTruth: string,
|
|
303
|
+
getAnswer: (modelId: string) => string
|
|
304
|
+
): AccuracyFunction {
|
|
305
|
+
return (subset: string[]) => {
|
|
306
|
+
if (subset.length === 0) return 0;
|
|
307
|
+
const votes: Record<string, number> = {};
|
|
308
|
+
for (const m of subset) {
|
|
309
|
+
const ans = getAnswer(m);
|
|
310
|
+
votes[ans] = (votes[ans] || 0) + 1;
|
|
311
|
+
}
|
|
312
|
+
const majority = Object.entries(votes).sort((a, b) => b[1] - a[1])[0]?.[0] || '';
|
|
313
|
+
return majority === groundTruth ? 1 : 0;
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** Apply Shapley credit to voting weights */
|
|
318
|
+
export function applyCredit(
|
|
319
|
+
contributions: Map<string, ModelContribution>,
|
|
320
|
+
baseWeights: Record<string, number>,
|
|
321
|
+
alpha = 0.5
|
|
322
|
+
): Record<string, number> {
|
|
323
|
+
const result: Record<string, number> = {};
|
|
324
|
+
for (const [model, contrib] of contributions) {
|
|
325
|
+
const base = baseWeights[model] || 1.0;
|
|
326
|
+
result[model] = (1 - alpha) * base + alpha * contrib.combinedCredit;
|
|
327
|
+
}
|
|
328
|
+
const sum = Object.values(result).reduce((a, b) => a + b, 0);
|
|
329
|
+
for (const k in result) result[k] /= sum;
|
|
330
|
+
return result;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
export interface ShapleySummary {
|
|
334
|
+
totalCredit: number;
|
|
335
|
+
perModel: ModelContribution[];
|
|
336
|
+
bestContributor: string;
|
|
337
|
+
worstContributor: string;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
export function summarize(
|
|
341
|
+
contributions: Map<string, ModelContribution>
|
|
342
|
+
): ShapleySummary {
|
|
343
|
+
const sorted = [...contributions.values()].sort((a, b) => b.combinedCredit - a.combinedCredit);
|
|
344
|
+
return {
|
|
345
|
+
totalCredit: sorted.reduce((s, c) => s + c.combinedCredit, 0),
|
|
346
|
+
perModel: sorted,
|
|
347
|
+
bestContributor: sorted[0]?.modelId || 'none',
|
|
348
|
+
worstContributor: sorted[sorted.length - 1]?.modelId || 'none',
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// Default export
|
|
353
|
+
export default { calculateEnhancedShapley, LoyaltyMatrix, HandicapCalculator, createAccuracyFn, applyCredit, summarize };
|
package/src/ensemble.ts
CHANGED
|
@@ -1,4 +1,14 @@
|
|
|
1
1
|
import { createA3MRouter } from './index';
|
|
2
|
+
import {
|
|
3
|
+
calculateEnhancedShapley,
|
|
4
|
+
LoyaltyMatrix,
|
|
5
|
+
HandicapCalculator,
|
|
6
|
+
createAccuracyFn,
|
|
7
|
+
applyCredit,
|
|
8
|
+
summarize,
|
|
9
|
+
ShapleySummary
|
|
10
|
+
} from './ensemble/shapleyValue';
|
|
11
|
+
import { dialogOptimizer, MultiRoundDialogOptimizer } from './ensemble/multiRoundDialog';
|
|
2
12
|
|
|
3
13
|
// RouterDecision type
|
|
4
14
|
interface RouteDecision {
|
|
@@ -17,7 +27,7 @@ export type RouterDecision = RouteDecision;
|
|
|
17
27
|
export const A3MRouter = createA3MRouter as any;
|
|
18
28
|
export { createA3MRouter };
|
|
19
29
|
|
|
20
|
-
export type EnsembleStrategy = 'majority' | 'weighted' | 'conservative';
|
|
30
|
+
export type EnsembleStrategy = 'majority' | 'weighted' | 'conservative' | 'shapley';
|
|
21
31
|
|
|
22
32
|
export interface EnsembleResponse {
|
|
23
33
|
finalAnswer: string;
|
|
@@ -26,24 +36,26 @@ export interface EnsembleResponse {
|
|
|
26
36
|
winner: string;
|
|
27
37
|
allResults: Record<string, { answer: string; score: number }>;
|
|
28
38
|
reasoning: string;
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
interface AnswerCount {
|
|
32
|
-
answer: string;
|
|
33
|
-
count: number;
|
|
39
|
+
shapleySummary?: ShapleySummary; // NEW: Shapley credit breakdown
|
|
40
|
+
dialogState?: ReturnType<typeof dialogOptimizer.getSummary>; // NEW: Dialog context
|
|
34
41
|
}
|
|
35
42
|
|
|
36
43
|
export class EnsembleOrchestrator {
|
|
44
|
+
private loyaltyMatrix = new LoyaltyMatrix();
|
|
45
|
+
private handicapCalc = new HandicapCalculator();
|
|
46
|
+
|
|
37
47
|
constructor(private router: InstanceType<typeof A3MRouter>) {}
|
|
38
48
|
|
|
39
49
|
/**
|
|
40
|
-
*
|
|
50
|
+
* Execute ensemble with enhanced Shapley value credit assignment
|
|
51
|
+
* Incorporates ethnocentrism (loyalty) and handicap (costly signaling)
|
|
41
52
|
*/
|
|
42
53
|
async executeEnsemble(
|
|
43
54
|
query: string,
|
|
44
55
|
providers: string[],
|
|
45
56
|
strategy: EnsembleStrategy = 'majority',
|
|
46
|
-
weights: Record<string, number> = {}
|
|
57
|
+
weights: Record<string, number> = {},
|
|
58
|
+
dialogId?: string
|
|
47
59
|
): Promise<EnsembleResponse> {
|
|
48
60
|
// 1. Parallel Execution
|
|
49
61
|
const results = await Promise.all(
|
|
@@ -58,9 +70,7 @@ export class EnsembleOrchestrator {
|
|
|
58
70
|
);
|
|
59
71
|
|
|
60
72
|
const successful = results.filter(r => r.success);
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
if (answers.length === 0) {
|
|
73
|
+
if (successful.length === 0) {
|
|
64
74
|
throw new Error('All ensemble providers failed.');
|
|
65
75
|
}
|
|
66
76
|
|
|
@@ -68,34 +78,37 @@ export class EnsembleOrchestrator {
|
|
|
68
78
|
let winnerAnswer = '';
|
|
69
79
|
let winnerProvider = '';
|
|
70
80
|
let confidence = 0;
|
|
81
|
+
let shapleySummary: ShapleySummary | undefined;
|
|
82
|
+
|
|
83
|
+
// Multi-round: add user turn
|
|
84
|
+
if (dialogId) dialogOptimizer.addTurn(dialogId, 'user', query);
|
|
71
85
|
|
|
72
86
|
if (strategy === 'majority') {
|
|
73
87
|
const counts: Record<string, number> = {};
|
|
74
88
|
successful.forEach(r => { counts[r.answer] = (counts[r.answer] || 0) + 1; });
|
|
75
|
-
const sorted
|
|
89
|
+
const sorted = Object.entries(counts).sort((a, b) => b[1] - a[1]);
|
|
76
90
|
winnerAnswer = sorted[0][0];
|
|
77
|
-
confidence = sorted[0][1] /
|
|
91
|
+
confidence = sorted[0][1] / successful.length;
|
|
78
92
|
winnerProvider = successful.find(r => r.answer === winnerAnswer)?.provider || 'unknown';
|
|
79
93
|
}
|
|
80
94
|
else if (strategy === 'weighted') {
|
|
81
95
|
const weightedCounts: Record<string, number> = {};
|
|
82
96
|
successful.forEach(r => {
|
|
83
|
-
const
|
|
84
|
-
weightedCounts[r.answer] = (weightedCounts[r.answer] || 0) +
|
|
97
|
+
const w = weights[r.provider] || 1.0;
|
|
98
|
+
weightedCounts[r.answer] = (weightedCounts[r.answer] || 0) + w;
|
|
85
99
|
});
|
|
86
|
-
const sorted
|
|
100
|
+
const sorted = Object.entries(weightedCounts).sort((a, b) => b[1] - a[1]);
|
|
87
101
|
winnerAnswer = sorted[0][0];
|
|
88
|
-
confidence = sorted[0][1] /
|
|
102
|
+
confidence = sorted[0][1] / successful.length;
|
|
89
103
|
winnerProvider = successful.find(r => r.answer === winnerAnswer)?.provider || 'unknown';
|
|
90
104
|
}
|
|
91
105
|
else if (strategy === 'conservative') {
|
|
92
106
|
const counts: Record<string, number> = {};
|
|
93
107
|
successful.forEach(r => { counts[r.answer] = (counts[r.answer] || 0) + 1; });
|
|
94
|
-
const best
|
|
95
|
-
|
|
108
|
+
const best = Object.entries(counts).sort((a, b) => b[1] - a[1])[0];
|
|
96
109
|
if (best && best[1] >= 2) {
|
|
97
110
|
winnerAnswer = best[0];
|
|
98
|
-
confidence = best[1] /
|
|
111
|
+
confidence = best[1] / successful.length;
|
|
99
112
|
winnerProvider = successful.find(r => r.answer === winnerAnswer)?.provider || 'unknown';
|
|
100
113
|
} else {
|
|
101
114
|
winnerAnswer = 'UNCERTAIN';
|
|
@@ -103,23 +116,101 @@ export class EnsembleOrchestrator {
|
|
|
103
116
|
winnerProvider = 'none';
|
|
104
117
|
}
|
|
105
118
|
}
|
|
119
|
+
else if (strategy === 'shapley') {
|
|
120
|
+
// === ENHANCED SHAPLEY WITH ETHNOCENTRISM + HANDICAP ===
|
|
121
|
+
const providerIds = successful.map(r => r.provider);
|
|
122
|
+
|
|
123
|
+
// Accuracy function based on majority vote as ground truth proxy
|
|
124
|
+
const accFn = createAccuracyFn(
|
|
125
|
+
winnerAnswer || successful[0].answer, // Will be updated after first pass
|
|
126
|
+
m => successful.find(r => r.provider === m)?.answer || ''
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
// Calculate enhanced Shapley with loyalty and handicap
|
|
130
|
+
const contributions = calculateEnhancedShapley(
|
|
131
|
+
providerIds,
|
|
132
|
+
accFn,
|
|
133
|
+
this.loyaltyMatrix,
|
|
134
|
+
this.handicapCalc
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
// Get majority vote using Shapley-weighted voting
|
|
138
|
+
const shapleyWeights = applyCredit(contributions, weights, 0.5);
|
|
139
|
+
const weightedCounts: Record<string, number> = {};
|
|
140
|
+
successful.forEach(r => {
|
|
141
|
+
weightedCounts[r.answer] = (weightedCounts[r.answer] || 0) + shapleyWeights[r.provider];
|
|
142
|
+
});
|
|
143
|
+
const sorted = Object.entries(weightedCounts).sort((a, b) => b[1] - a[1]);
|
|
144
|
+
winnerAnswer = sorted[0][0];
|
|
145
|
+
confidence = sorted[0][1];
|
|
146
|
+
winnerProvider = successful.find(r => r.answer === winnerAnswer)?.provider || 'unknown';
|
|
147
|
+
|
|
148
|
+
// Update Shapley summary
|
|
149
|
+
shapleySummary = summarize(contributions);
|
|
150
|
+
|
|
151
|
+
// Record performance for handicap tracking
|
|
152
|
+
const isCorrect = (ans: string) => ans === winnerAnswer;
|
|
153
|
+
successful.forEach(r => {
|
|
154
|
+
const cost = weights[r.provider] || 0.001;
|
|
155
|
+
this.handicapCalc.record(r.provider, cost, isCorrect(r.answer));
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Record loyalty: successful collaborations build trust
|
|
160
|
+
if (strategy === 'shapley') {
|
|
161
|
+
for (const r of successful) {
|
|
162
|
+
if (r.answer === winnerAnswer) {
|
|
163
|
+
for (const other of successful) {
|
|
164
|
+
if (other.provider !== r.provider && other.answer === winnerAnswer) {
|
|
165
|
+
this.loyaltyMatrix.recordSuccess(r.provider, other.provider, 1.0);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Multi-round: add assistant turn
|
|
173
|
+
if (dialogId && winnerAnswer !== 'UNCERTAIN') {
|
|
174
|
+
dialogOptimizer.addTurn(dialogId, 'assistant', winnerAnswer, winnerProvider);
|
|
175
|
+
}
|
|
106
176
|
|
|
107
177
|
// 3. Final Assembly
|
|
108
178
|
const allResults: Record<string, { answer: string; score: number }> = {};
|
|
109
179
|
successful.forEach(r => {
|
|
110
|
-
allResults[r.provider] = {
|
|
111
|
-
answer: r.answer,
|
|
112
|
-
score: r.answer === winnerAnswer ? 1.0 : 0.0
|
|
113
|
-
};
|
|
180
|
+
allResults[r.provider] = { answer: r.answer, score: r.answer === winnerAnswer ? 1.0 : 0.0 };
|
|
114
181
|
});
|
|
115
182
|
|
|
183
|
+
let dialogState;
|
|
184
|
+
if (dialogId) dialogState = dialogOptimizer.getSummary(dialogId);
|
|
185
|
+
|
|
116
186
|
return {
|
|
117
187
|
finalAnswer: winnerAnswer,
|
|
118
|
-
confidence
|
|
188
|
+
confidence,
|
|
119
189
|
isUncertain: confidence < 0.6 || winnerAnswer === 'UNCERTAIN',
|
|
120
190
|
winner: winnerProvider,
|
|
121
191
|
allResults,
|
|
122
|
-
reasoning: `Ensemble of ${successful.length} models. ${Math.round(confidence * 100)}% agreement
|
|
192
|
+
reasoning: `Ensemble of ${successful.length} models. ${Math.round(confidence * 100)}% agreement.`,
|
|
193
|
+
shapleySummary,
|
|
194
|
+
dialogState,
|
|
123
195
|
};
|
|
124
196
|
}
|
|
197
|
+
|
|
198
|
+
/** Get best model for current dialog topic */
|
|
199
|
+
getBestModelForTopic(dialogId: string, availableModels: string[]): string | null {
|
|
200
|
+
return dialogOptimizer.getBestModelForTopic(dialogId, availableModels);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Build optimized context for multi-turn conversation */
|
|
204
|
+
buildOptimizedContext(dialogId: string, newQuery: string): string {
|
|
205
|
+
return dialogOptimizer.buildOptimizedContext(dialogId, newQuery);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Clear dialog state */
|
|
209
|
+
clearDialog(dialogId: string): void {
|
|
210
|
+
dialogOptimizer.clearState(dialogId);
|
|
211
|
+
}
|
|
125
212
|
}
|
|
213
|
+
|
|
214
|
+
// Re-export enhanced utilities
|
|
215
|
+
export { LoyaltyMatrix, HandicapCalculator, calculateEnhancedShapley } from './ensemble/shapleyValue';
|
|
216
|
+
export { dialogOptimizer, MultiRoundDialogOptimizer } from './ensemble/multiRoundDialog';
|
package/src/index.ts
CHANGED
|
@@ -104,6 +104,30 @@ export type { Span, Metric, RouteTrace, ObservabilityEvent } from './observabili
|
|
|
104
104
|
// ============================================================
|
|
105
105
|
export { EnsembleOrchestrator, EnsembleStrategy, EnsembleResponse } from './ensemble';
|
|
106
106
|
|
|
107
|
+
// ============================================================
|
|
108
|
+
// ENHANCED SHAPLEY VALUE (Game Theory: Ethnocentrism + Handicap)
|
|
109
|
+
// ============================================================
|
|
110
|
+
export {
|
|
111
|
+
LoyaltyMatrix,
|
|
112
|
+
HandicapCalculator,
|
|
113
|
+
calculateEnhancedShapley,
|
|
114
|
+
createAccuracyFn,
|
|
115
|
+
applyCredit,
|
|
116
|
+
summarize,
|
|
117
|
+
type ShapleyConfig,
|
|
118
|
+
type ShapleySummary,
|
|
119
|
+
} from './ensemble/shapleyValue';
|
|
120
|
+
|
|
121
|
+
// ============================================================
|
|
122
|
+
// MULTI-ROUND DIALOG OPTIMIZATION
|
|
123
|
+
// ============================================================
|
|
124
|
+
export {
|
|
125
|
+
MultiRoundDialogOptimizer,
|
|
126
|
+
dialogOptimizer,
|
|
127
|
+
type DialogTurn,
|
|
128
|
+
type DialogState,
|
|
129
|
+
} from './ensemble/multiRoundDialog';
|
|
130
|
+
|
|
107
131
|
// ============================================================
|
|
108
132
|
// SCIENCE ADAPTER (Google DeepMind Skills)
|
|
109
133
|
// ============================================================
|
|
@@ -42,13 +42,13 @@ const DOMAIN_SKILLS: Record<string, string[]> = {
|
|
|
42
42
|
};
|
|
43
43
|
|
|
44
44
|
// Pre-built research prompts for common science queries
|
|
45
|
-
const RESEARCH_TEMPLATES: Record<string, string> = {
|
|
45
|
+
export const RESEARCH_TEMPLATES: Record<string, string> = {
|
|
46
46
|
protein_structure: 'What is the 3D structure of {protein}? Show the AlphaFold prediction.',
|
|
47
47
|
gene_function: 'What is the biological function of the {gene} gene in {species}?',
|
|
48
48
|
disease_genes: 'What genes are associated with {disease}? List with relevance scores.',
|
|
49
49
|
drug_interactions: 'What drugs interact with target {protein}? Include binding affinities.',
|
|
50
50
|
pathway_analysis: 'What biological pathways involve {gene}? Show interactions.',
|
|
51
|
-
|
|
51
|
+
'variant_pathogenicity': 'What is the pathogenicity of variant {variant} in {gene}?',
|
|
52
52
|
expression_levels: 'What is the expression pattern of {gene} in {tissue}?',
|
|
53
53
|
literature_review: 'Summarize recent literature on {topic}. Include key findings.',
|
|
54
54
|
};
|