@cogitator-ai/core 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +920 -15
- package/dist/cogitator.d.ts +31 -1
- package/dist/cogitator.d.ts.map +1 -1
- package/dist/cogitator.js +127 -6
- package/dist/cogitator.js.map +1 -1
- package/dist/constitutional/constitution.d.ts +9 -0
- package/dist/constitutional/constitution.d.ts.map +1 -0
- package/dist/constitutional/constitution.js +215 -0
- package/dist/constitutional/constitution.js.map +1 -0
- package/dist/constitutional/constitutional-ai.d.ts +36 -0
- package/dist/constitutional/constitutional-ai.d.ts.map +1 -0
- package/dist/constitutional/constitutional-ai.js +163 -0
- package/dist/constitutional/constitutional-ai.js.map +1 -0
- package/dist/constitutional/critique-reviser.d.ts +20 -0
- package/dist/constitutional/critique-reviser.d.ts.map +1 -0
- package/dist/constitutional/critique-reviser.js +98 -0
- package/dist/constitutional/critique-reviser.js.map +1 -0
- package/dist/constitutional/index.d.ts +13 -0
- package/dist/constitutional/index.d.ts.map +1 -0
- package/dist/constitutional/index.js +8 -0
- package/dist/constitutional/index.js.map +1 -0
- package/dist/constitutional/input-filter.d.ts +19 -0
- package/dist/constitutional/input-filter.d.ts.map +1 -0
- package/dist/constitutional/input-filter.js +88 -0
- package/dist/constitutional/input-filter.js.map +1 -0
- package/dist/constitutional/output-filter.d.ts +19 -0
- package/dist/constitutional/output-filter.d.ts.map +1 -0
- package/dist/constitutional/output-filter.js +86 -0
- package/dist/constitutional/output-filter.js.map +1 -0
- package/dist/constitutional/prompts.d.ts +11 -0
- package/dist/constitutional/prompts.d.ts.map +1 -0
- package/dist/constitutional/prompts.js +202 -0
- package/dist/constitutional/prompts.js.map +1 -0
- package/dist/constitutional/tool-guard.d.ts +18 -0
- package/dist/constitutional/tool-guard.d.ts.map +1 -0
- package/dist/constitutional/tool-guard.js +125 -0
- package/dist/constitutional/tool-guard.js.map +1 -0
- package/dist/cost-routing/budget-enforcer.d.ts +26 -0
- package/dist/cost-routing/budget-enforcer.d.ts.map +1 -0
- package/dist/cost-routing/budget-enforcer.js +86 -0
- package/dist/cost-routing/budget-enforcer.js.map +1 -0
- package/dist/cost-routing/cost-router.d.ts +34 -0
- package/dist/cost-routing/cost-router.d.ts.map +1 -0
- package/dist/cost-routing/cost-router.js +80 -0
- package/dist/cost-routing/cost-router.js.map +1 -0
- package/dist/cost-routing/cost-tracker.d.ts +20 -0
- package/dist/cost-routing/cost-tracker.d.ts.map +1 -0
- package/dist/cost-routing/cost-tracker.js +85 -0
- package/dist/cost-routing/cost-tracker.js.map +1 -0
- package/dist/cost-routing/index.d.ts +6 -0
- package/dist/cost-routing/index.d.ts.map +1 -0
- package/dist/cost-routing/index.js +6 -0
- package/dist/cost-routing/index.js.map +1 -0
- package/dist/cost-routing/model-selector.d.ts +15 -0
- package/dist/cost-routing/model-selector.d.ts.map +1 -0
- package/dist/cost-routing/model-selector.js +216 -0
- package/dist/cost-routing/model-selector.js.map +1 -0
- package/dist/cost-routing/task-analyzer.d.ts +13 -0
- package/dist/cost-routing/task-analyzer.d.ts.map +1 -0
- package/dist/cost-routing/task-analyzer.js +185 -0
- package/dist/cost-routing/task-analyzer.js.map +1 -0
- package/dist/index.d.ts +13 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/learning/ab-testing.d.ts +45 -0
- package/dist/learning/ab-testing.d.ts.map +1 -0
- package/dist/learning/ab-testing.js +267 -0
- package/dist/learning/ab-testing.js.map +1 -0
- package/dist/learning/agent-optimizer.d.ts.map +1 -1
- package/dist/learning/agent-optimizer.js +26 -21
- package/dist/learning/agent-optimizer.js.map +1 -1
- package/dist/learning/auto-optimizer.d.ts +38 -0
- package/dist/learning/auto-optimizer.d.ts.map +1 -0
- package/dist/learning/auto-optimizer.js +229 -0
- package/dist/learning/auto-optimizer.js.map +1 -0
- package/dist/learning/demo-selector.d.ts.map +1 -1
- package/dist/learning/demo-selector.js +7 -7
- package/dist/learning/demo-selector.js.map +1 -1
- package/dist/learning/index.d.ts +13 -1
- package/dist/learning/index.d.ts.map +1 -1
- package/dist/learning/index.js +7 -1
- package/dist/learning/index.js.map +1 -1
- package/dist/learning/instruction-optimizer.d.ts.map +1 -1
- package/dist/learning/instruction-optimizer.js +7 -11
- package/dist/learning/instruction-optimizer.js.map +1 -1
- package/dist/learning/metrics.d.ts.map +1 -1
- package/dist/learning/metrics.js +26 -16
- package/dist/learning/metrics.js.map +1 -1
- package/dist/learning/postgres-trace-store.d.ts +53 -0
- package/dist/learning/postgres-trace-store.d.ts.map +1 -0
- package/dist/learning/postgres-trace-store.js +692 -0
- package/dist/learning/postgres-trace-store.js.map +1 -0
- package/dist/learning/prompt-logger.d.ts +29 -0
- package/dist/learning/prompt-logger.d.ts.map +1 -0
- package/dist/learning/prompt-logger.js +157 -0
- package/dist/learning/prompt-logger.js.map +1 -0
- package/dist/learning/prompt-monitor.d.ts +29 -0
- package/dist/learning/prompt-monitor.d.ts.map +1 -0
- package/dist/learning/prompt-monitor.js +243 -0
- package/dist/learning/prompt-monitor.js.map +1 -0
- package/dist/learning/prompts.d.ts.map +1 -1
- package/dist/learning/prompts.js +24 -13
- package/dist/learning/prompts.js.map +1 -1
- package/dist/learning/rollback-manager.d.ts +36 -0
- package/dist/learning/rollback-manager.d.ts.map +1 -0
- package/dist/learning/rollback-manager.js +177 -0
- package/dist/learning/rollback-manager.js.map +1 -0
- package/dist/learning/trace-store.d.ts.map +1 -1
- package/dist/learning/trace-store.js +8 -10
- package/dist/learning/trace-store.js.map +1 -1
- package/dist/reasoning/branch-evaluator.d.ts.map +1 -1
- package/dist/reasoning/branch-evaluator.js +14 -8
- package/dist/reasoning/branch-evaluator.js.map +1 -1
- package/dist/reasoning/branch-generator.d.ts.map +1 -1
- package/dist/reasoning/branch-generator.js +5 -3
- package/dist/reasoning/branch-generator.js.map +1 -1
- package/dist/reasoning/prompts.d.ts.map +1 -1
- package/dist/reasoning/prompts.js +7 -5
- package/dist/reasoning/prompts.js.map +1 -1
- package/dist/reasoning/thought-tree.d.ts.map +1 -1
- package/dist/reasoning/thought-tree.js +9 -11
- package/dist/reasoning/thought-tree.js.map +1 -1
- package/dist/reflection/insight-store.d.ts.map +1 -1
- package/dist/reflection/insight-store.js +8 -6
- package/dist/reflection/insight-store.js.map +1 -1
- package/dist/reflection/prompts.d.ts.map +1 -1
- package/dist/reflection/prompts.js +11 -6
- package/dist/reflection/prompts.js.map +1 -1
- package/dist/reflection/reflection-engine.d.ts.map +1 -1
- package/dist/reflection/reflection-engine.js +8 -10
- package/dist/reflection/reflection-engine.js.map +1 -1
- package/dist/time-travel/checkpoint-store.d.ts +34 -0
- package/dist/time-travel/checkpoint-store.d.ts.map +1 -0
- package/dist/time-travel/checkpoint-store.js +240 -0
- package/dist/time-travel/checkpoint-store.js.map +1 -0
- package/dist/time-travel/comparator.d.ts +26 -0
- package/dist/time-travel/comparator.d.ts.map +1 -0
- package/dist/time-travel/comparator.js +253 -0
- package/dist/time-travel/comparator.js.map +1 -0
- package/dist/time-travel/forker.d.ts +22 -0
- package/dist/time-travel/forker.d.ts.map +1 -0
- package/dist/time-travel/forker.js +118 -0
- package/dist/time-travel/forker.js.map +1 -0
- package/dist/time-travel/index.d.ts +6 -0
- package/dist/time-travel/index.d.ts.map +1 -0
- package/dist/time-travel/index.js +6 -0
- package/dist/time-travel/index.js.map +1 -0
- package/dist/time-travel/replayer.d.ts +20 -0
- package/dist/time-travel/replayer.d.ts.map +1 -0
- package/dist/time-travel/replayer.js +147 -0
- package/dist/time-travel/replayer.js.map +1 -0
- package/dist/time-travel/time-travel.d.ts +41 -0
- package/dist/time-travel/time-travel.d.ts.map +1 -0
- package/dist/time-travel/time-travel.js +127 -0
- package/dist/time-travel/time-travel.js.map +1 -0
- package/package.json +13 -5
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
const DEFAULT_VALUES = {
|
|
2
|
+
enabled: false,
|
|
3
|
+
triggerAfterRuns: 100,
|
|
4
|
+
minRunsForOptimization: 20,
|
|
5
|
+
requireABTest: true,
|
|
6
|
+
maxOptimizationsPerDay: 3,
|
|
7
|
+
};
|
|
8
|
+
export class AutoOptimizer {
|
|
9
|
+
config;
|
|
10
|
+
agentOptimizer;
|
|
11
|
+
abTesting;
|
|
12
|
+
monitor;
|
|
13
|
+
rollbackManager;
|
|
14
|
+
runCounts = new Map();
|
|
15
|
+
dailyOptimizations = new Map();
|
|
16
|
+
activeRuns = new Map();
|
|
17
|
+
constructor(config) {
|
|
18
|
+
this.config = {
|
|
19
|
+
enabled: config.enabled ?? DEFAULT_VALUES.enabled,
|
|
20
|
+
triggerAfterRuns: config.triggerAfterRuns ?? DEFAULT_VALUES.triggerAfterRuns,
|
|
21
|
+
minRunsForOptimization: config.minRunsForOptimization ?? DEFAULT_VALUES.minRunsForOptimization,
|
|
22
|
+
requireABTest: config.requireABTest ?? DEFAULT_VALUES.requireABTest,
|
|
23
|
+
maxOptimizationsPerDay: config.maxOptimizationsPerDay ?? DEFAULT_VALUES.maxOptimizationsPerDay,
|
|
24
|
+
agentOptimizer: config.agentOptimizer,
|
|
25
|
+
abTesting: config.abTesting,
|
|
26
|
+
monitor: config.monitor,
|
|
27
|
+
rollbackManager: config.rollbackManager,
|
|
28
|
+
onOptimizationStart: config.onOptimizationStart,
|
|
29
|
+
onOptimizationComplete: config.onOptimizationComplete,
|
|
30
|
+
onRollback: config.onRollback,
|
|
31
|
+
};
|
|
32
|
+
this.agentOptimizer = config.agentOptimizer;
|
|
33
|
+
this.abTesting = config.abTesting;
|
|
34
|
+
this.monitor = config.monitor;
|
|
35
|
+
this.rollbackManager = config.rollbackManager;
|
|
36
|
+
}
|
|
37
|
+
async recordExecution(trace) {
|
|
38
|
+
if (!this.config.enabled)
|
|
39
|
+
return;
|
|
40
|
+
const agentId = trace.agentId;
|
|
41
|
+
const count = (this.runCounts.get(agentId) ?? 0) + 1;
|
|
42
|
+
this.runCounts.set(agentId, count);
|
|
43
|
+
const alerts = this.monitor.recordExecution(trace);
|
|
44
|
+
await this.handleAlerts(agentId, alerts);
|
|
45
|
+
const activeABTest = await this.abTesting.getActiveTest(agentId);
|
|
46
|
+
if (activeABTest && trace.score !== undefined) {
|
|
47
|
+
const currentVersion = await this.rollbackManager.getCurrentVersion(agentId);
|
|
48
|
+
const variant = currentVersion?.instructions === activeABTest.treatmentInstructions ? 'treatment' : 'control';
|
|
49
|
+
await this.abTesting.recordResult(activeABTest.id, variant, trace.score, trace.duration ?? 0, trace.usage?.cost ?? 0);
|
|
50
|
+
const outcome = await this.abTesting.checkAndCompleteIfReady(activeABTest.id);
|
|
51
|
+
if (outcome) {
|
|
52
|
+
await this.handleABTestCompletion(agentId, activeABTest.id, outcome);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (trace.score !== undefined) {
|
|
56
|
+
await this.rollbackManager.recordMetrics(agentId, trace.score, trace.duration ?? 0, trace.usage?.cost ?? 0, trace.metrics?.success ?? true);
|
|
57
|
+
}
|
|
58
|
+
if (count >= this.config.triggerAfterRuns && await this.shouldTriggerOptimization(agentId)) {
|
|
59
|
+
await this.triggerOptimization(agentId);
|
|
60
|
+
this.runCounts.set(agentId, 0);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
async triggerOptimization(agentId) {
|
|
64
|
+
const run = this.createOptimizationRun(agentId);
|
|
65
|
+
this.activeRuns.set(agentId, run);
|
|
66
|
+
if (this.config.onOptimizationStart) {
|
|
67
|
+
this.config.onOptimizationStart(run);
|
|
68
|
+
}
|
|
69
|
+
try {
|
|
70
|
+
run.status = 'optimizing';
|
|
71
|
+
const currentVersion = await this.rollbackManager.getCurrentVersion(agentId);
|
|
72
|
+
const currentInstructions = currentVersion?.instructions ?? '';
|
|
73
|
+
const agent = { id: agentId, instructions: currentInstructions };
|
|
74
|
+
const optimizationResult = await this.agentOptimizer.compile(agent, [], {
|
|
75
|
+
maxRounds: 2,
|
|
76
|
+
optimizeInstructions: true,
|
|
77
|
+
});
|
|
78
|
+
if (!optimizationResult.success || optimizationResult.improvement <= 0) {
|
|
79
|
+
run.status = 'completed';
|
|
80
|
+
run.completedAt = new Date();
|
|
81
|
+
run.error = optimizationResult.errors.join('; ') || 'No improvement found';
|
|
82
|
+
if (this.config.onOptimizationComplete) {
|
|
83
|
+
this.config.onOptimizationComplete(run);
|
|
84
|
+
}
|
|
85
|
+
return run;
|
|
86
|
+
}
|
|
87
|
+
const newInstructions = optimizationResult.instructionsAfter ?? currentInstructions;
|
|
88
|
+
if (this.config.requireABTest) {
|
|
89
|
+
run.status = 'testing';
|
|
90
|
+
const abTest = await this.abTesting.createTest({
|
|
91
|
+
agentId,
|
|
92
|
+
name: `Auto-optimization ${new Date().toISOString()}`,
|
|
93
|
+
controlInstructions: currentInstructions,
|
|
94
|
+
treatmentInstructions: newInstructions,
|
|
95
|
+
});
|
|
96
|
+
run.abTestId = abTest.id;
|
|
97
|
+
await this.abTesting.startTest(abTest.id);
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
run.status = 'deploying';
|
|
101
|
+
const newVersion = await this.rollbackManager.deployVersion(agentId, newInstructions, 'optimization', run.id);
|
|
102
|
+
run.deployedVersionId = newVersion.id;
|
|
103
|
+
run.status = 'completed';
|
|
104
|
+
run.completedAt = new Date();
|
|
105
|
+
}
|
|
106
|
+
this.incrementDailyCount(agentId);
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
run.status = 'failed';
|
|
110
|
+
run.completedAt = new Date();
|
|
111
|
+
run.error = error instanceof Error ? error.message : String(error);
|
|
112
|
+
}
|
|
113
|
+
if (run.status === 'completed' || run.status === 'failed') {
|
|
114
|
+
if (this.config.onOptimizationComplete) {
|
|
115
|
+
this.config.onOptimizationComplete(run);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return run;
|
|
119
|
+
}
|
|
120
|
+
async forceRollback(agentId, reason) {
|
|
121
|
+
if (!this.monitor.canRollback(agentId)) {
|
|
122
|
+
return false;
|
|
123
|
+
}
|
|
124
|
+
const result = await this.rollbackManager.rollbackToPrevious(agentId);
|
|
125
|
+
if (result.success) {
|
|
126
|
+
this.monitor.recordRollback(agentId);
|
|
127
|
+
this.monitor.clearWindow(agentId);
|
|
128
|
+
const activeRun = this.activeRuns.get(agentId);
|
|
129
|
+
if (activeRun && activeRun.status !== 'completed') {
|
|
130
|
+
activeRun.status = 'rolled_back';
|
|
131
|
+
activeRun.completedAt = new Date();
|
|
132
|
+
activeRun.error = reason;
|
|
133
|
+
if (this.config.onOptimizationComplete) {
|
|
134
|
+
this.config.onOptimizationComplete(activeRun);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
if (this.config.onRollback) {
|
|
138
|
+
this.config.onRollback(agentId, reason);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return result.success;
|
|
142
|
+
}
|
|
143
|
+
getActiveRun(agentId) {
|
|
144
|
+
return this.activeRuns.get(agentId) ?? null;
|
|
145
|
+
}
|
|
146
|
+
getRunCount(agentId) {
|
|
147
|
+
return this.runCounts.get(agentId) ?? 0;
|
|
148
|
+
}
|
|
149
|
+
getDailyOptimizationCount(agentId) {
|
|
150
|
+
const today = new Date().toISOString().split('T')[0];
|
|
151
|
+
const daily = this.dailyOptimizations.get(agentId);
|
|
152
|
+
if (daily?.date !== today) {
|
|
153
|
+
return 0;
|
|
154
|
+
}
|
|
155
|
+
return daily.count;
|
|
156
|
+
}
|
|
157
|
+
setEnabled(enabled) {
|
|
158
|
+
this.config.enabled = enabled;
|
|
159
|
+
}
|
|
160
|
+
async shouldTriggerOptimization(agentId) {
|
|
161
|
+
if (!this.config.enabled)
|
|
162
|
+
return false;
|
|
163
|
+
const activeRun = this.activeRuns.get(agentId);
|
|
164
|
+
if (activeRun && !['completed', 'failed', 'rolled_back'].includes(activeRun.status)) {
|
|
165
|
+
return false;
|
|
166
|
+
}
|
|
167
|
+
const activeABTest = await this.abTesting.getActiveTest(agentId);
|
|
168
|
+
if (activeABTest) {
|
|
169
|
+
return false;
|
|
170
|
+
}
|
|
171
|
+
const dailyCount = this.getDailyOptimizationCount(agentId);
|
|
172
|
+
if (dailyCount >= this.config.maxOptimizationsPerDay) {
|
|
173
|
+
return false;
|
|
174
|
+
}
|
|
175
|
+
return true;
|
|
176
|
+
}
|
|
177
|
+
async handleAlerts(agentId, alerts) {
|
|
178
|
+
for (const alert of alerts) {
|
|
179
|
+
if (alert.severity === 'critical' && alert.autoAction === 'rollback') {
|
|
180
|
+
const rolled = await this.forceRollback(agentId, `Critical alert: ${alert.type}`);
|
|
181
|
+
if (rolled) {
|
|
182
|
+
alert.actionTaken = true;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
async handleABTestCompletion(agentId, testId, outcome) {
|
|
188
|
+
const activeRun = this.activeRuns.get(agentId);
|
|
189
|
+
if (activeRun?.abTestId !== testId) {
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
activeRun.abTestOutcome = outcome;
|
|
193
|
+
if (outcome.winner === 'treatment') {
|
|
194
|
+
activeRun.status = 'deploying';
|
|
195
|
+
const test = await this.abTesting.getActiveTest(agentId);
|
|
196
|
+
if (test) {
|
|
197
|
+
const newVersion = await this.rollbackManager.deployVersion(agentId, test.treatmentInstructions, 'ab_test', testId);
|
|
198
|
+
activeRun.deployedVersionId = newVersion.id;
|
|
199
|
+
}
|
|
200
|
+
activeRun.status = 'completed';
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
activeRun.status = 'completed';
|
|
204
|
+
}
|
|
205
|
+
activeRun.completedAt = new Date();
|
|
206
|
+
if (this.config.onOptimizationComplete) {
|
|
207
|
+
this.config.onOptimizationComplete(activeRun);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
createOptimizationRun(agentId) {
|
|
211
|
+
return {
|
|
212
|
+
id: `opt-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
|
|
213
|
+
agentId,
|
|
214
|
+
status: 'pending',
|
|
215
|
+
startedAt: new Date(),
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
incrementDailyCount(agentId) {
|
|
219
|
+
const today = new Date().toISOString().split('T')[0];
|
|
220
|
+
const current = this.dailyOptimizations.get(agentId);
|
|
221
|
+
if (current?.date !== today) {
|
|
222
|
+
this.dailyOptimizations.set(agentId, { date: today, count: 1 });
|
|
223
|
+
}
|
|
224
|
+
else {
|
|
225
|
+
current.count++;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
//# sourceMappingURL=auto-optimizer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auto-optimizer.js","sourceRoot":"","sources":["../../src/learning/auto-optimizer.ts"],"names":[],"mappings":"AAuCA,MAAM,cAAc,GAAG;IACrB,OAAO,EAAE,KAAK;IACd,gBAAgB,EAAE,GAAG;IACrB,sBAAsB,EAAE,EAAE;IAC1B,aAAa,EAAE,IAAI;IACnB,sBAAsB,EAAE,CAAC;CAC1B,CAAC;AAEF,MAAM,OAAO,aAAa;IAChB,MAAM,CAAwB;IAC9B,cAAc,CAAiB;IAC/B,SAAS,CAAqB;IAC9B,OAAO,CAAgB;IACvB,eAAe,CAAkB;IAEjC,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,kBAAkB,GAAG,IAAI,GAAG,EAA2C,CAAC;IACxE,UAAU,GAAG,IAAI,GAAG,EAA2B,CAAC;IAExD,YAAY,MAA2B;QACrC,IAAI,CAAC,MAAM,GAAG;YACZ,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,cAAc,CAAC,OAAO;YACjD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,cAAc,CAAC,gBAAgB;YAC5E,sBAAsB,EAAE,MAAM,CAAC,sBAAsB,IAAI,cAAc,CAAC,sBAAsB;YAC9F,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC,aAAa;YACnE,sBAAsB,EAAE,MAAM,CAAC,sBAAsB,IAAI,cAAc,CAAC,sBAAsB;YAC9F,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;YAC/C,sBAAsB,EAAE,MAAM,CAAC,sBAAsB;YACrD,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAAqB;QACzC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO;QAEjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;QAE9B,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAEnC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAEzC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACjE,IAAI,YAAY,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAC7E,MAAM,OAAO,GACX,cAAc,EAAE,YAAY,KAAK,YAAY,CAAC,qBAAqB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YAEhG,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAC/B,YAAY,CAAC,EAAE,EACf,OAAO,EACP,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,QAAQ,IAAI,CAAC,EACnB,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CACvB,CAAC;YAEF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAC9E,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAED,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CACtC,OAAO,EACP,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,QAAQ,IAAI,CAAC,EACnB,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,EACtB,KAAK,CAAC,OAAO,EAAE,OAAO,IAAI,IAAI,CAC/B,CAAC;QACJ,CAAC;QAED,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,MAAM,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3F,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAAe;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,CAAC;YACH,GAAG,CAAC,MAAM,GAAG,YAAY,CAAC;YAE1B,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAC7E,MAAM,mBAAmB,GAAG,cAAc,EAAE,YAAY,IAAI,EAAE,CAAC;YAE/D,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAW,CAAC;YAC1E,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE;gBACtE,SAAS,EAAE,CAAC;gBACZ,oBAAoB,EAAE,IAAI;aAC3B,CAAC,CAAC;YAEH,IAAI,CAAC,kBAAkB,CAAC,OAAO,IAAI,kBAAkB,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;gBACvE,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;gBACzB,GAAG,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;gBAC7B,GAAG,CAAC,KAAK,GAAG,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,sBAAsB,CAAC;gBAE3E,IAAI,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;oBACvC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;gBAC1C,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC;YAED,MAAM,eAAe,GAAG,kBAAkB,CAAC,iBAAiB,IAAI,mBAAmB,CAAC;YAEpF,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC9B,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;gBAEvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;oBAC7C,OAAO;oBACP,IAAI,EAAE,qBAAqB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE;oBACrD,mBAAmB,EAAE,mBAAmB;oBACxC,qBAAqB,EAAE,eAAe;iBACvC,CAAC,CAAC;gBAEH,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC5C,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;gBAEzB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CACzD,OAAO,EACP,eAAe,EACf,cAAc,EACd,GAAG,CAAC,EAAE,CACP,CAAC;gBAEF,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC,EAAE,CAAC;gBACtC,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;gBACzB,GAAG,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;YAC/B,CAAC;YAED,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAEpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC;YACtB,GAAG,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;YAC7B,GAAG,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrE,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC1D,IAAI,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;gBACvC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,MAAc;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEtE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAElC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClD,SAAS,CAAC,MAAM,GAAG,aAAa,CAAC;gBACjC,SAAS,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;gBACnC,SAAS,CAAC,KAAK,GAAG,MAAM,CAAC;gBAEzB,IAAI,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;oBACvC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;gBAC3B,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,YAAY,CAAC,OAAe;QAC1B,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC9C,CAAC;IAED,WAAW,CAAC,OAAe;QACzB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,yBAAyB,CAAC,OAAe;QACvC,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,KAAK,EAAE,IAAI,KAAK,KAAK,EAAE,CAAC;YAC1B,OAAO,CAAC,CAAC;QACX,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IACrB,CAAC;IAED,UAAU,CAAC,OAAgB;QACzB,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;IAChC,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,OAAe;QACrD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAEvC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,SAAS,IAAI,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACpF,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACjE,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAC3D,IAAI,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;YACrD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,OAAe,EAAE,MAA0B;QACpE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,QAAQ,KAAK,UAAU,IAAI,KAAK,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBACrE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,mBAAmB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;gBAClF,IAAI,MAAM,EAAE,CAAC;oBACX,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;gBAC3B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAClC,OAAe,EACf,MAAc,EACd,OAAsB;QAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,SAAS,EAAE,QAAQ,KAAK,MAAM,EAAE,CAAC;YACnC,OAAO;QACT,CAAC;QAED,SAAS,CAAC,aAAa,GAAG,OAAO,CAAC;QAElC,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YACnC,SAAS,CAAC,MAAM,GAAG,WAAW,CAAC;YAE/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACzD,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CACzD,OAAO,EACP,IAAI,CAAC,qBAAqB,EAC1B,SAAS,EACT,MAAM,CACP,CAAC;gBACF,SAAS,CAAC,iBAAiB,GAAG,UAAU,CAAC,EAAE,CAAC;YAC9C,CAAC;YAED,SAAS,CAAC,MAAM,GAAG,WAAW,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,MAAM,GAAG,WAAW,CAAC;QACjC,CAAC;QAED,SAAS,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;QAEnC,IAAI,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAEO,qBAAqB,CAAC,OAAe;QAC3C,OAAO;YACL,EAAE,EAAE,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;YACrE,OAAO;YACP,MAAM,EAAE,SAAkC;YAC1C,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;IACJ,CAAC;IAEO,mBAAmB,CAAC,OAAe;QACzC,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAErD,IAAI,OAAO,EAAE,IAAI,KAAK,KAAK,EAAE,CAAC;YAC5B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"demo-selector.d.ts","sourceRoot":"","sources":["../../src/learning/demo-selector.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"demo-selector.d.ts","sourceRoot":"","sources":["../../src/learning/demo-selector.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAY,SAAS,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjG,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,UAAU,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAA2B;IACxC,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAS;gBAEb,OAAO,EAAE,mBAAmB;IAMlC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAwC3E,OAAO,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAuC7C,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrC,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAuCvD,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IAIhC,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,EAAE;IAapC,OAAO,CAAC,kBAAkB;IAa1B,OAAO,CAAC,kBAAkB;IAoC1B,OAAO,CAAC,mBAAmB;IAa3B,OAAO,CAAC,eAAe;IAiBvB,OAAO,CAAC,cAAc;YAeR,kBAAkB;IAsBhC,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM;CAgB5C"}
|
|
@@ -22,8 +22,8 @@ export class DemoSelector {
|
|
|
22
22
|
demos.push(demo);
|
|
23
23
|
}
|
|
24
24
|
const inputLower = input.toLowerCase();
|
|
25
|
-
const inputWords = new Set(inputLower.split(/\s+/).filter(w => w.length > 3));
|
|
26
|
-
const scored = demos.map(demo => {
|
|
25
|
+
const inputWords = new Set(inputLower.split(/\s+/).filter((w) => w.length > 3));
|
|
26
|
+
const scored = demos.map((demo) => {
|
|
27
27
|
const relevance = this.calculateRelevance(inputWords, demo);
|
|
28
28
|
const quality = demo.score;
|
|
29
29
|
const usage = Math.min(demo.usageCount * 0.1, 0.5);
|
|
@@ -31,7 +31,7 @@ export class DemoSelector {
|
|
|
31
31
|
return { demo, score };
|
|
32
32
|
});
|
|
33
33
|
scored.sort((a, b) => b.score - a.score);
|
|
34
|
-
const selected = this.diversifySelection(scored.map(s => s.demo), limit);
|
|
34
|
+
const selected = this.diversifySelection(scored.map((s) => s.demo), limit);
|
|
35
35
|
for (const demo of selected) {
|
|
36
36
|
demo.usageCount++;
|
|
37
37
|
demo.lastUsedAt = new Date();
|
|
@@ -97,9 +97,9 @@ export class DemoSelector {
|
|
|
97
97
|
const totalScore = demos.reduce((sum, d) => sum + d.score, 0);
|
|
98
98
|
const averageScore = totalScore / demos.length;
|
|
99
99
|
const usageDistribution = demos
|
|
100
|
-
.map(d => ({ demoId: d.id, count: d.usageCount }))
|
|
100
|
+
.map((d) => ({ demoId: d.id, count: d.usageCount }))
|
|
101
101
|
.sort((a, b) => b.count - a.count);
|
|
102
|
-
const unusedDemos = demos.filter(d => d.usageCount === 0);
|
|
102
|
+
const unusedDemos = demos.filter((d) => d.usageCount === 0);
|
|
103
103
|
const coverageGaps = unusedDemos.length > 0
|
|
104
104
|
? ['Some demos have never been used - may indicate coverage gaps']
|
|
105
105
|
: [];
|
|
@@ -190,7 +190,7 @@ export class DemoSelector {
|
|
|
190
190
|
extractContext(trace) {
|
|
191
191
|
const parts = [];
|
|
192
192
|
if (trace.toolCalls.length > 0) {
|
|
193
|
-
const tools = [...new Set(trace.toolCalls.map(t => t.name))];
|
|
193
|
+
const tools = [...new Set(trace.toolCalls.map((t) => t.name))];
|
|
194
194
|
parts.push(`Tools: ${tools.join(', ')}`);
|
|
195
195
|
}
|
|
196
196
|
if (trace.labels && trace.labels.length > 0) {
|
|
@@ -223,7 +223,7 @@ export class DemoSelector {
|
|
|
223
223
|
return '';
|
|
224
224
|
const formatted = demos.map((demo, i) => {
|
|
225
225
|
const stepsStr = demo.keySteps.length > 0
|
|
226
|
-
? `\nSteps: ${demo.keySteps.map(s => s.description).join(' → ')}`
|
|
226
|
+
? `\nSteps: ${demo.keySteps.map((s) => s.description).join(' → ')}`
|
|
227
227
|
: '';
|
|
228
228
|
return `Example ${i + 1}:
|
|
229
229
|
Input: ${demo.input}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"demo-selector.js","sourceRoot":"","sources":["../../src/learning/demo-selector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"demo-selector.js","sourceRoot":"","sources":["../../src/learning/demo-selector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,MAAM,OAAO,YAAY;IACf,UAAU,CAAa;IACvB,KAAK,GAAG,IAAI,GAAG,EAAgB,CAAC;IAChC,UAAU,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,QAAQ,CAAS;IACjB,QAAQ,CAAS;IAEzB,YAAY,OAA4B;QACtC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,OAAe,EAAE,KAAa,EAAE,KAAa;QAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,KAAK,GAAW,EAAE,CAAC;QACzB,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChC,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAEhF,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAChC,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;YAEnD,MAAM,KAAK,GAAG,SAAS,GAAG,GAAG,GAAG,OAAO,GAAG,GAAG,GAAG,KAAK,GAAG,GAAG,CAAC;YAE5D,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAEzC,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CACtC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EACzB,KAAK,CACN,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAqB;QACjC,IAAI,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,eAAe,KAAK,CAAC,KAAK,kBAAkB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,MAAM,IAAI,GAAS;YACjB,EAAE,EAAE,QAAQ,MAAM,CAAC,EAAE,CAAC,EAAE;YACxB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,OAAO,EAAE,KAAK,CAAC,EAAE;YACjB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,QAAQ,EAAE,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YACrC,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC;SACpC,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAE9B,IAAI,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACtD,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,YAAY,GAAG,IAAI,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACnD,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE1B,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAE3C,IAAI,YAAY,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/C,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,EAAU;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAE9C,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,OAAe;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAElD,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC7C,OAAO;gBACL,UAAU,EAAE,CAAC;gBACb,YAAY,EAAE,CAAC;gBACf,iBAAiB,EAAE,EAAE;gBACrB,YAAY,EAAE,EAAE;aACjB,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAW,EAAE,CAAC;QACzB,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChC,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9D,MAAM,YAAY,GAAG,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;QAE/C,MAAM,iBAAiB,GAAG,KAAK;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;aACnD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAErC,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC;QAC5D,MAAM,YAAY,GAChB,WAAW,CAAC,MAAM,GAAG,CAAC;YACpB,CAAC,CAAC,CAAC,8DAA8D,CAAC;YAClE,CAAC,CAAC,EAAE,CAAC;QAET,OAAO;YACL,UAAU,EAAE,KAAK,CAAC,MAAM;YACxB,YAAY;YACZ,iBAAiB;YACjB,YAAY;SACb,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,EAAU;QAChB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC;IACpC,CAAC;IAED,WAAW,CAAC,OAAe;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,CAAC,YAAY;YAAE,OAAO,EAAE,CAAC;QAE7B,MAAM,KAAK,GAAW,EAAE,CAAC;QACzB,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChC,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACjD,CAAC;IAEO,kBAAkB,CAAC,UAAuB,EAAE,IAAU;QAC5D,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC,WAAW,EAAE,CAAC;QAEpF,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IAEO,kBAAkB,CAAC,KAAa,EAAE,KAAa;QACrD,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;YAC1B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,QAAQ,GAAW,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QAE7B,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAG,CAAC,CAAC;QAElC,OAAO,QAAQ,CAAC,MAAM,GAAG,KAAK,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,aAAa,GAAG,CAAC,CAAC,CAAC;YAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC/B,IAAI,aAAa,GAAG,CAAC,CAAC;gBAEtB,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;oBAChC,MAAM,UAAU,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;oBACjE,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;gBACtD,CAAC;gBAED,MAAM,SAAS,GAAG,CAAC,GAAG,aAAa,CAAC;gBACpC,IAAI,SAAS,GAAG,aAAa,EAAE,CAAC;oBAC9B,aAAa,GAAG,SAAS,CAAC;oBAC1B,SAAS,GAAG,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,mBAAmB,CAAC,CAAO,EAAE,CAAO;QAC1C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAE3D,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,YAAY,EAAE,CAAC;QACvC,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,GAAG,YAAY,CAAC;QACvD,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IAEO,eAAe,CAAC,KAAqB;QAC3C,MAAM,QAAQ,GAAe,EAAE,CAAC;QAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC/C,QAAQ,CAAC,IAAI,CAAC;oBACZ,WAAW,EAAE,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,OAAO;oBAC9C,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;oBAC5B,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;oBAClC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM;iBACpC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9B,CAAC;IAEO,cAAc,CAAC,KAAqB;QAC1C,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/D,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,OAAe;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAEhE,MAAM,KAAK,GAAW,EAAE,CAAC;QACzB,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChC,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClB,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;YACjE,OAAO,MAAM,GAAG,MAAM,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,oBAAoB,CAAC,KAAa;QAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;YACtC,MAAM,QAAQ,GACZ,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;gBACtB,CAAC,CAAC,YAAY,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBACnE,CAAC,CAAC,EAAE,CAAC;YAET,OAAO,WAAW,CAAC,GAAG,CAAC;SACpB,IAAI,CAAC,KAAK;UACT,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,OAAO,2DAA2D,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;IAC7F,CAAC;CACF"}
|
package/dist/learning/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { InMemoryTraceStore } from './trace-store';
|
|
2
|
-
export { MetricEvaluator, createSuccessMetric, createExactMatchMetric, createContainsMetric } from './metrics';
|
|
2
|
+
export { MetricEvaluator, createSuccessMetric, createExactMatchMetric, createContainsMetric, } from './metrics';
|
|
3
3
|
export type { MetricEvaluatorOptions } from './metrics';
|
|
4
4
|
export { DemoSelector } from './demo-selector';
|
|
5
5
|
export type { DemoSelectorOptions } from './demo-selector';
|
|
@@ -9,4 +9,16 @@ export { AgentOptimizer } from './agent-optimizer';
|
|
|
9
9
|
export type { AgentOptimizerOptions } from './agent-optimizer';
|
|
10
10
|
export { buildFailureAnalysisPrompt, buildInstructionCandidatePrompt, buildInstructionEvaluationPrompt, buildInstructionRefinementPrompt, parseFailureAnalysisResponse, parseInstructionCandidatesResponse, parseInstructionEvaluationResponse, parseInstructionRefinementResponse, } from './prompts';
|
|
11
11
|
export type { FailureAnalysisResult, InstructionCandidate, InstructionEvaluation, InstructionRefinement, } from './prompts';
|
|
12
|
+
export { PostgresTraceStore } from './postgres-trace-store';
|
|
13
|
+
export type { PostgresTraceStoreConfig } from './postgres-trace-store';
|
|
14
|
+
export { PromptLogger, wrapWithPromptLogger } from './prompt-logger';
|
|
15
|
+
export type { PromptLoggerContext, PromptLoggerConfig } from './prompt-logger';
|
|
16
|
+
export { ABTestingFramework } from './ab-testing';
|
|
17
|
+
export type { ABTestingFrameworkConfig } from './ab-testing';
|
|
18
|
+
export { PromptMonitor } from './prompt-monitor';
|
|
19
|
+
export type { PromptMonitorConfig } from './prompt-monitor';
|
|
20
|
+
export { RollbackManager } from './rollback-manager';
|
|
21
|
+
export type { RollbackManagerConfig, RollbackResult } from './rollback-manager';
|
|
22
|
+
export { AutoOptimizer } from './auto-optimizer';
|
|
23
|
+
export type { AutoOptimizerConfig } from './auto-optimizer';
|
|
12
24
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/learning/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/learning/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,sBAAsB,EACtB,oBAAoB,GACrB,MAAM,WAAW,CAAC;AACnB,YAAY,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,YAAY,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EACL,0BAA0B,EAC1B,+BAA+B,EAC/B,gCAAgC,EAChC,gCAAgC,EAChC,4BAA4B,EAC5B,kCAAkC,EAClC,kCAAkC,EAClC,kCAAkC,GACnC,MAAM,WAAW,CAAC;AACnB,YAAY,EACV,qBAAqB,EACrB,oBAAoB,EACpB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,WAAW,CAAC;AAEnB,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,YAAY,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACrE,YAAY,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,YAAY,EAAE,wBAAwB,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,YAAY,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,YAAY,EAAE,qBAAqB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,YAAY,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/learning/index.js
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
export { InMemoryTraceStore } from './trace-store';
|
|
2
|
-
export { MetricEvaluator, createSuccessMetric, createExactMatchMetric, createContainsMetric } from './metrics';
|
|
2
|
+
export { MetricEvaluator, createSuccessMetric, createExactMatchMetric, createContainsMetric, } from './metrics';
|
|
3
3
|
export { DemoSelector } from './demo-selector';
|
|
4
4
|
export { InstructionOptimizer } from './instruction-optimizer';
|
|
5
5
|
export { AgentOptimizer } from './agent-optimizer';
|
|
6
6
|
export { buildFailureAnalysisPrompt, buildInstructionCandidatePrompt, buildInstructionEvaluationPrompt, buildInstructionRefinementPrompt, parseFailureAnalysisResponse, parseInstructionCandidatesResponse, parseInstructionEvaluationResponse, parseInstructionRefinementResponse, } from './prompts';
|
|
7
|
+
export { PostgresTraceStore } from './postgres-trace-store';
|
|
8
|
+
export { PromptLogger, wrapWithPromptLogger } from './prompt-logger';
|
|
9
|
+
export { ABTestingFramework } from './ab-testing';
|
|
10
|
+
export { PromptMonitor } from './prompt-monitor';
|
|
11
|
+
export { RollbackManager } from './rollback-manager';
|
|
12
|
+
export { AutoOptimizer } from './auto-optimizer';
|
|
7
13
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/learning/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/learning/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,sBAAsB,EACtB,oBAAoB,GACrB,MAAM,WAAW,CAAC;AAEnB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EACL,0BAA0B,EAC1B,+BAA+B,EAC/B,gCAAgC,EAChC,gCAAgC,EAChC,4BAA4B,EAC5B,kCAAkC,EAClC,kCAAkC,EAClC,kCAAkC,GACnC,MAAM,WAAW,CAAC;AAQnB,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAErE,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAElD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"instruction-optimizer.d.ts","sourceRoot":"","sources":["../../src/learning/instruction-optimizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,UAAU,EACV,YAAY,EAGZ,6BAA6B,EAC7B,UAAU,EACX,MAAM,qBAAqB,CAAC;AAY7B,MAAM,WAAW,2BAA2B;IAC1C,GAAG,EAAE,UAAU,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,UAAU,CAAC;IACvB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,GAAG,CAAa;IACxB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,gBAAgB,CAAS;gBAErB,OAAO,EAAE,2BAA2B;IAS1C,QAAQ,CACZ,OAAO,EAAE,MAAM,EACf,mBAAmB,EAAE,MAAM,EAC3B,OAAO,CAAC,EAAE;QACR,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;QAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GACA,OAAO,CAAC,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"instruction-optimizer.d.ts","sourceRoot":"","sources":["../../src/learning/instruction-optimizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,UAAU,EACV,YAAY,EAGZ,6BAA6B,EAC7B,UAAU,EACX,MAAM,qBAAqB,CAAC;AAY7B,MAAM,WAAW,2BAA2B;IAC1C,GAAG,EAAE,UAAU,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,UAAU,CAAC;IACvB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,GAAG,CAAa;IACxB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,gBAAgB,CAAS;gBAErB,OAAO,EAAE,2BAA2B;IAS1C,QAAQ,CACZ,OAAO,EAAE,MAAM,EACf,mBAAmB,EAAE,MAAM,EAC3B,OAAO,CAAC,EAAE;QACR,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;QAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GACA,OAAO,CAAC,6BAA6B,CAAC;YAoF3B,eAAe;YAqBf,kBAAkB;YAyBlB,kBAAkB;YAkClB,kBAAkB;YAqBlB,wBAAwB;CAgBvC"}
|
|
@@ -15,7 +15,7 @@ export class InstructionOptimizer {
|
|
|
15
15
|
this.refinementRounds = options.refinementRounds ?? 1;
|
|
16
16
|
}
|
|
17
17
|
async optimize(agentId, currentInstructions, options) {
|
|
18
|
-
const traces = options?.traces ?? await this.traceStore.getAll(agentId);
|
|
18
|
+
const traces = options?.traces ?? (await this.traceStore.getAll(agentId));
|
|
19
19
|
const maxTraces = options?.maxTraces ?? 20;
|
|
20
20
|
const relevantTraces = traces.slice(0, maxTraces);
|
|
21
21
|
if (relevantTraces.length === 0) {
|
|
@@ -111,8 +111,8 @@ export class InstructionOptimizer {
|
|
|
111
111
|
});
|
|
112
112
|
const parsed = parseInstructionCandidatesResponse(response.content);
|
|
113
113
|
return parsed
|
|
114
|
-
.map(c => c.instructions)
|
|
115
|
-
.filter(i => i.length > 0)
|
|
114
|
+
.map((c) => c.instructions)
|
|
115
|
+
.filter((i) => i.length > 0)
|
|
116
116
|
.slice(0, this.candidateCount);
|
|
117
117
|
}
|
|
118
118
|
catch {
|
|
@@ -164,16 +164,12 @@ export class InstructionOptimizer {
|
|
|
164
164
|
}
|
|
165
165
|
}
|
|
166
166
|
async estimateInstructionScore(instructions, traces) {
|
|
167
|
-
const successfulTraces = traces.filter(t => t.metrics.success);
|
|
168
|
-
const baseScore = traces.length > 0
|
|
169
|
-
|
|
170
|
-
: 0.5;
|
|
171
|
-
const avgTraceScore = traces.length > 0
|
|
172
|
-
? traces.reduce((sum, t) => sum + t.score, 0) / traces.length
|
|
173
|
-
: 0.5;
|
|
167
|
+
const successfulTraces = traces.filter((t) => t.metrics.success);
|
|
168
|
+
const baseScore = traces.length > 0 ? successfulTraces.length / traces.length : 0.5;
|
|
169
|
+
const avgTraceScore = traces.length > 0 ? traces.reduce((sum, t) => sum + t.score, 0) / traces.length : 0.5;
|
|
174
170
|
const instructionLength = instructions.length;
|
|
175
171
|
const conciseBonus = instructionLength < 500 ? 0.1 : instructionLength > 1500 ? -0.1 : 0;
|
|
176
|
-
return Math.max(0, Math.min(1,
|
|
172
|
+
return Math.max(0, Math.min(1, baseScore * 0.4 + avgTraceScore * 0.5 + 0.5 + conciseBonus));
|
|
177
173
|
}
|
|
178
174
|
}
|
|
179
175
|
//# sourceMappingURL=instruction-optimizer.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"instruction-optimizer.js","sourceRoot":"","sources":["../../src/learning/instruction-optimizer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,0BAA0B,EAC1B,+BAA+B,EAC/B,gCAAgC,EAChC,gCAAgC,EAChC,4BAA4B,EAC5B,kCAAkC,EAClC,kCAAkC,EAClC,kCAAkC,GACnC,MAAM,WAAW,CAAC;AAWnB,MAAM,OAAO,oBAAoB;IACvB,GAAG,CAAa;IAChB,KAAK,CAAS;IACd,UAAU,CAAa;IACvB,YAAY,CAAgB;IAC5B,cAAc,CAAS;IACvB,gBAAgB,CAAS;IAEjC,YAAY,OAAoC;QAC9C,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,OAAe,EACf,mBAA2B,EAC3B,OAIC;QAED,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"instruction-optimizer.js","sourceRoot":"","sources":["../../src/learning/instruction-optimizer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,0BAA0B,EAC1B,+BAA+B,EAC/B,gCAAgC,EAChC,gCAAgC,EAChC,4BAA4B,EAC5B,kCAAkC,EAClC,kCAAkC,EAClC,kCAAkC,GACnC,MAAM,WAAW,CAAC;AAWnB,MAAM,OAAO,oBAAoB;IACvB,GAAG,CAAa;IAChB,KAAK,CAAS;IACd,UAAU,CAAa;IACvB,YAAY,CAAgB;IAC5B,cAAc,CAAS;IACvB,gBAAgB,CAAS;IAEjC,YAAY,OAAoC;QAC9C,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,OAAe,EACf,mBAA2B,EAC3B,OAIC;QAED,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1E,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;QAC3C,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAElD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO;gBACL,oBAAoB,EAAE,mBAAmB;gBACzC,qBAAqB,EAAE,mBAAmB;gBAC1C,WAAW,EAAE,CAAC;gBACd,aAAa,EAAE,EAAE;gBACjB,mBAAmB,EAAE,CAAC;gBACtB,SAAS,EAAE,sCAAsC;aAClD,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;QAE7E,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO;gBACL,oBAAoB,EAAE,mBAAmB;gBACzC,qBAAqB,EAAE,mBAAmB;gBAC1C,WAAW,EAAE,CAAC;gBACd,aAAa,EAAE,EAAE;gBACjB,mBAAmB,EAAE,CAAC;gBACtB,SAAS,EAAE,gCAAgC;aAC5C,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY;YAChC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,OAAO,EAAE,mBAAmB,EAAE,EAAE,CAAC;YACxE,CAAC,CAAC,EAAE,CAAC;QAEP,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,mBAAmB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;QAEtF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;gBACL,oBAAoB,EAAE,mBAAmB;gBACzC,qBAAqB,EAAE,mBAAmB;gBAC1C,WAAW,EAAE,CAAC;gBACd,aAAa,EAAE,IAAI;gBACnB,mBAAmB,EAAE,CAAC;gBACtB,SAAS,EAAE,2CAA2C;aACvD,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAE9E,IAAI,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,cAAc,GAAG,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;QAEpF,KAAK,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,WAAW,EAAE,CAAC;YAClD,IAAI,UAAU,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;gBACjC,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC;gBAC7B,aAAa,GAAG,SAAS,CAAC;gBAC1B,cAAc,GAAG,UAAU,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,IAAI,iBAAiB,GAAG,aAAa,CAAC;QACtC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3D,IAAI,cAAc,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAElD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;YAE5F,IAAI,OAAO,EAAE,CAAC;gBACZ,iBAAiB,GAAG,OAAO,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;QAC/F,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;QAExF,OAAO;YACL,oBAAoB,EAAE,mBAAmB;YACzC,qBAAqB,EAAE,iBAAiB;YACxC,WAAW,EAAE,QAAQ,GAAG,aAAa;YACrC,aAAa,EAAE,IAAI;YACnB,mBAAmB,EAAE,UAAU,CAAC,MAAM;YACtC,SAAS,EAAE,cAAc,IAAI,CAAC,MAAM,oBAAoB,UAAU,CAAC,MAAM,4BAA4B,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;SAC5H,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,MAAwB,EACxB,mBAA2B;QAE3B,MAAM,MAAM,GAAG,0BAA0B,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;QAEvE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC7C,WAAW,EAAE,GAAG;gBAChB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,4BAA4B,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC9D,OAAO,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,mBAA2B,EAC3B,IAAsB,EACtB,QAAmB;QAEnB,MAAM,MAAM,GAAG,+BAA+B,CAAC,mBAAmB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;QAEpF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC7C,WAAW,EAAE,GAAG;gBAChB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kCAAkC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpE,OAAO,MAAM;iBACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;iBAC1B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;iBAC3B,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,UAAoB,EACpB,MAAwB;QAExB,MAAM,WAAW,GAAG,IAAI,GAAG,EAAmD,CAAC;QAE/E,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,gCAAgC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAEnE,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;oBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;oBAC7C,WAAW,EAAE,GAAG;oBAChB,SAAS,EAAE,GAAG;iBACf,CAAC,CAAC;gBAEH,MAAM,MAAM,GAAG,kCAAkC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpE,IAAI,MAAM,EAAE,CAAC;oBACX,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE;wBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;wBACnB,UAAU,EAAE,MAAM,CAAC,UAAU;qBAC9B,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,SAAiB,EACjB,UAAoB;QAEpB,MAAM,MAAM,GAAG,gCAAgC,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAEvE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC7C,WAAW,EAAE,GAAG;gBAChB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kCAAkC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpE,OAAO,MAAM,EAAE,YAAY,IAAI,IAAI,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,YAAoB,EACpB,MAAwB;QAExB,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEjE,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;QAEpF,MAAM,aAAa,GACjB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;QAExF,MAAM,iBAAiB,GAAG,YAAY,CAAC,MAAM,CAAC;QAC9C,MAAM,YAAY,GAAG,iBAAiB,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEzF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,GAAG,GAAG,aAAa,GAAG,GAAG,GAAG,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC;IAC9F,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/learning/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,QAAQ,EACR,qBAAqB,EAErB,UAAU,EACX,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,sBAAsB;IACrC,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC;CACzC;
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/learning/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,QAAQ,EACR,qBAAqB,EAErB,UAAU,EACX,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,sBAAsB;IACrC,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC;CACzC;AAsBD,qBAAa,eAAe;IAC1B,OAAO,CAAC,GAAG,CAAC,CAAa;IACzB,OAAO,CAAC,KAAK,CAAC,CAAS;IACvB,OAAO,CAAC,MAAM,CAAwB;IACtC,OAAO,CAAC,aAAa,CAA+B;gBAExC,OAAO,GAAE,sBAA2B;IAMhD,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,GAAG,IAAI;IAI1C,QAAQ,CACZ,KAAK,EAAE,cAAc,EACrB,QAAQ,CAAC,EAAE,OAAO,GACjB,OAAO,CAAC;QAAE,OAAO,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IAcjE,aAAa,CACjB,MAAM,EAAE,cAAc,EAAE,EACxB,YAAY,CAAC,EAAE,OAAO,EAAE,GACvB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;YAatE,cAAc;IA0B5B,aAAa,CAAC,KAAK,EAAE,cAAc,GAAG,YAAY;IAmBlD,kBAAkB,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,YAAY;IAwC3E,gBAAgB,CAAC,KAAK,EAAE,cAAc,GAAG,YAAY;IAiB/C,kBAAkB,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC;IAiDpF,eAAe,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;IAgDnE,OAAO,CAAC,mBAAmB;IAkB3B,OAAO,CAAC,eAAe;IA6BvB,SAAS,IAAI,qBAAqB;CAGnC;AAED,wBAAgB,mBAAmB,IAAI,QAAQ,CAS9C;AAED,wBAAgB,sBAAsB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,CAkBnE;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,QAAQ,CAoBjE"}
|
package/dist/learning/metrics.js
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
const DEFAULT_CONFIG = {
|
|
2
2
|
metrics: [
|
|
3
|
-
{
|
|
4
|
-
|
|
3
|
+
{
|
|
4
|
+
name: 'success',
|
|
5
|
+
type: 'boolean',
|
|
6
|
+
description: 'Did the run complete without errors?',
|
|
7
|
+
weight: 0.4,
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
name: 'tool_accuracy',
|
|
11
|
+
type: 'numeric',
|
|
12
|
+
description: 'Did tools produce expected results?',
|
|
13
|
+
weight: 0.3,
|
|
14
|
+
},
|
|
5
15
|
{ name: 'efficiency', type: 'numeric', description: 'Token/time efficiency', weight: 0.3 },
|
|
6
16
|
],
|
|
7
17
|
aggregation: 'weighted-average',
|
|
@@ -61,17 +71,20 @@ export class MetricEvaluator {
|
|
|
61
71
|
}
|
|
62
72
|
}
|
|
63
73
|
successMetric(trace) {
|
|
64
|
-
const hasErrors = trace.steps.some(step => step.toolResult?.error ||
|
|
74
|
+
const hasErrors = trace.steps.some((step) => step.toolResult?.error ||
|
|
75
|
+
(step.type === 'reflection' && step.reflection?.analysis?.wasSuccessful === false));
|
|
65
76
|
const value = hasErrors ? 0 : 1;
|
|
66
77
|
return {
|
|
67
78
|
name: 'success',
|
|
68
79
|
value,
|
|
69
80
|
passed: value === 1,
|
|
70
|
-
reasoning: hasErrors
|
|
81
|
+
reasoning: hasErrors
|
|
82
|
+
? 'Run had errors or failed reflections'
|
|
83
|
+
: 'Run completed without errors',
|
|
71
84
|
};
|
|
72
85
|
}
|
|
73
86
|
toolAccuracyMetric(trace, expected) {
|
|
74
|
-
const toolSteps = trace.steps.filter(s => s.type === 'tool_call');
|
|
87
|
+
const toolSteps = trace.steps.filter((s) => s.type === 'tool_call');
|
|
75
88
|
if (toolSteps.length === 0) {
|
|
76
89
|
return {
|
|
77
90
|
name: 'tool_accuracy',
|
|
@@ -109,7 +122,7 @@ export class MetricEvaluator {
|
|
|
109
122
|
const duration = trace.duration;
|
|
110
123
|
const tokenEfficiency = Math.min(1, 10000 / Math.max(totalTokens, 1));
|
|
111
124
|
const timeEfficiency = Math.min(1, 30000 / Math.max(duration, 1));
|
|
112
|
-
const value =
|
|
125
|
+
const value = tokenEfficiency * 0.6 + timeEfficiency * 0.4;
|
|
113
126
|
return {
|
|
114
127
|
name: 'efficiency',
|
|
115
128
|
value,
|
|
@@ -175,7 +188,7 @@ Respond with JSON: { "score": 0.X, "reasoning": "..." }`;
|
|
|
175
188
|
const prompt = `Evaluate the logical coherence of this agent execution.
|
|
176
189
|
|
|
177
190
|
Input: ${trace.input}
|
|
178
|
-
Steps taken: ${trace.steps.map(s => s.type === 'tool_call' ? `Tool: ${s.toolCall?.name}` : s.type).join(' → ')}
|
|
191
|
+
Steps taken: ${trace.steps.map((s) => (s.type === 'tool_call' ? `Tool: ${s.toolCall?.name}` : s.type)).join(' → ')}
|
|
179
192
|
Output: ${trace.output}
|
|
180
193
|
|
|
181
194
|
Rate coherence from 0.0 to 1.0 where:
|
|
@@ -211,7 +224,7 @@ Respond with JSON: { "score": 0.X, "reasoning": "..." }`;
|
|
|
211
224
|
parseMetricResponse(content) {
|
|
212
225
|
try {
|
|
213
226
|
let jsonStr = content;
|
|
214
|
-
const codeBlockMatch =
|
|
227
|
+
const codeBlockMatch = /```(?:json)?\s*([\s\S]*?)\s*```/.exec(content);
|
|
215
228
|
if (codeBlockMatch) {
|
|
216
229
|
jsonStr = codeBlockMatch[1];
|
|
217
230
|
}
|
|
@@ -232,7 +245,7 @@ Respond with JSON: { "score": 0.X, "reasoning": "..." }`;
|
|
|
232
245
|
let totalWeight = 0;
|
|
233
246
|
let weightedSum = 0;
|
|
234
247
|
for (const result of results) {
|
|
235
|
-
const metricDef = this.config.metrics.find(m => m.name === result.name);
|
|
248
|
+
const metricDef = this.config.metrics.find((m) => m.name === result.name);
|
|
236
249
|
const weight = metricDef?.weight ?? 1;
|
|
237
250
|
weightedSum += result.value * weight;
|
|
238
251
|
totalWeight += weight;
|
|
@@ -240,7 +253,7 @@ Respond with JSON: { "score": 0.X, "reasoning": "..." }`;
|
|
|
240
253
|
return totalWeight > 0 ? weightedSum / totalWeight : 0;
|
|
241
254
|
}
|
|
242
255
|
case 'min':
|
|
243
|
-
return Math.min(...results.map(r => r.value));
|
|
256
|
+
return Math.min(...results.map((r) => r.value));
|
|
244
257
|
case 'product':
|
|
245
258
|
return results.reduce((acc, r) => acc * r.value, 1);
|
|
246
259
|
default:
|
|
@@ -253,7 +266,7 @@ Respond with JSON: { "score": 0.X, "reasoning": "..." }`;
|
|
|
253
266
|
}
|
|
254
267
|
export function createSuccessMetric() {
|
|
255
268
|
return (trace) => {
|
|
256
|
-
const hasErrors = trace.steps.some(step => step.toolResult?.error);
|
|
269
|
+
const hasErrors = trace.steps.some((step) => step.toolResult?.error);
|
|
257
270
|
return {
|
|
258
271
|
name: 'success',
|
|
259
272
|
value: hasErrors ? 0 : 1,
|
|
@@ -266,11 +279,8 @@ export function createExactMatchMetric(fieldPath) {
|
|
|
266
279
|
if (expected === undefined) {
|
|
267
280
|
return { name: 'exact_match', value: 1, passed: true, reasoning: 'No expected value' };
|
|
268
281
|
}
|
|
269
|
-
const outputValue = fieldPath
|
|
270
|
-
|
|
271
|
-
: trace.output;
|
|
272
|
-
const matches = String(outputValue).toLowerCase().trim() ===
|
|
273
|
-
String(expected).toLowerCase().trim();
|
|
282
|
+
const outputValue = fieldPath ? trace.output : trace.output;
|
|
283
|
+
const matches = String(outputValue).toLowerCase().trim() === String(expected).toLowerCase().trim();
|
|
274
284
|
return {
|
|
275
285
|
name: 'exact_match',
|
|
276
286
|
value: matches ? 1 : 0,
|