@cogitator-ai/core 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +920 -15
- package/dist/__tests__/agent.test.js +2 -2
- package/dist/__tests__/agent.test.js.map +1 -1
- package/dist/__tests__/base64.test.js +1 -1
- package/dist/__tests__/base64.test.js.map +1 -1
- package/dist/__tests__/calculator.test.js +1 -1
- package/dist/__tests__/calculator.test.js.map +1 -1
- package/dist/__tests__/cogitator-memory.test.js +2 -2
- package/dist/__tests__/cogitator-memory.test.js.map +1 -1
- package/dist/__tests__/datetime.test.js +1 -1
- package/dist/__tests__/datetime.test.js.map +1 -1
- package/dist/__tests__/exec.test.js +1 -1
- package/dist/__tests__/exec.test.js.map +1 -1
- package/dist/__tests__/filesystem.test.js +1 -1
- package/dist/__tests__/filesystem.test.js.map +1 -1
- package/dist/__tests__/google-backend.test.js +1 -1
- package/dist/__tests__/google-backend.test.js.map +1 -1
- package/dist/__tests__/hash.test.js +1 -1
- package/dist/__tests__/hash.test.js.map +1 -1
- package/dist/__tests__/http.test.js +1 -1
- package/dist/__tests__/http.test.js.map +1 -1
- package/dist/__tests__/json.test.js +1 -1
- package/dist/__tests__/json.test.js.map +1 -1
- package/dist/__tests__/logger.test.js +1 -1
- package/dist/__tests__/logger.test.js.map +1 -1
- package/dist/__tests__/random.test.js +1 -1
- package/dist/__tests__/random.test.js.map +1 -1
- package/dist/__tests__/regex.test.js +1 -1
- package/dist/__tests__/regex.test.js.map +1 -1
- package/dist/__tests__/registry.test.js +2 -2
- package/dist/__tests__/registry.test.js.map +1 -1
- package/dist/__tests__/sleep.test.js +1 -1
- package/dist/__tests__/sleep.test.js.map +1 -1
- package/dist/__tests__/tool.test.js +1 -1
- package/dist/__tests__/tool.test.js.map +1 -1
- package/dist/__tests__/uuid.test.js +1 -1
- package/dist/__tests__/uuid.test.js.map +1 -1
- package/dist/cogitator.d.ts +46 -1
- package/dist/cogitator.d.ts.map +1 -1
- package/dist/cogitator.js +274 -17
- package/dist/cogitator.js.map +1 -1
- package/dist/constitutional/constitution.d.ts +9 -0
- package/dist/constitutional/constitution.d.ts.map +1 -0
- package/dist/constitutional/constitution.js +215 -0
- package/dist/constitutional/constitution.js.map +1 -0
- package/dist/constitutional/constitutional-ai.d.ts +36 -0
- package/dist/constitutional/constitutional-ai.d.ts.map +1 -0
- package/dist/constitutional/constitutional-ai.js +163 -0
- package/dist/constitutional/constitutional-ai.js.map +1 -0
- package/dist/constitutional/critique-reviser.d.ts +20 -0
- package/dist/constitutional/critique-reviser.d.ts.map +1 -0
- package/dist/constitutional/critique-reviser.js +98 -0
- package/dist/constitutional/critique-reviser.js.map +1 -0
- package/dist/constitutional/index.d.ts +13 -0
- package/dist/constitutional/index.d.ts.map +1 -0
- package/dist/constitutional/index.js +8 -0
- package/dist/constitutional/index.js.map +1 -0
- package/dist/constitutional/input-filter.d.ts +19 -0
- package/dist/constitutional/input-filter.d.ts.map +1 -0
- package/dist/constitutional/input-filter.js +88 -0
- package/dist/constitutional/input-filter.js.map +1 -0
- package/dist/constitutional/output-filter.d.ts +19 -0
- package/dist/constitutional/output-filter.d.ts.map +1 -0
- package/dist/constitutional/output-filter.js +86 -0
- package/dist/constitutional/output-filter.js.map +1 -0
- package/dist/constitutional/prompts.d.ts +11 -0
- package/dist/constitutional/prompts.d.ts.map +1 -0
- package/dist/constitutional/prompts.js +202 -0
- package/dist/constitutional/prompts.js.map +1 -0
- package/dist/constitutional/tool-guard.d.ts +18 -0
- package/dist/constitutional/tool-guard.d.ts.map +1 -0
- package/dist/constitutional/tool-guard.js +125 -0
- package/dist/constitutional/tool-guard.js.map +1 -0
- package/dist/cost-routing/budget-enforcer.d.ts +26 -0
- package/dist/cost-routing/budget-enforcer.d.ts.map +1 -0
- package/dist/cost-routing/budget-enforcer.js +86 -0
- package/dist/cost-routing/budget-enforcer.js.map +1 -0
- package/dist/cost-routing/cost-router.d.ts +34 -0
- package/dist/cost-routing/cost-router.d.ts.map +1 -0
- package/dist/cost-routing/cost-router.js +80 -0
- package/dist/cost-routing/cost-router.js.map +1 -0
- package/dist/cost-routing/cost-tracker.d.ts +20 -0
- package/dist/cost-routing/cost-tracker.d.ts.map +1 -0
- package/dist/cost-routing/cost-tracker.js +85 -0
- package/dist/cost-routing/cost-tracker.js.map +1 -0
- package/dist/cost-routing/index.d.ts +6 -0
- package/dist/cost-routing/index.d.ts.map +1 -0
- package/dist/cost-routing/index.js +6 -0
- package/dist/cost-routing/index.js.map +1 -0
- package/dist/cost-routing/model-selector.d.ts +15 -0
- package/dist/cost-routing/model-selector.d.ts.map +1 -0
- package/dist/cost-routing/model-selector.js +216 -0
- package/dist/cost-routing/model-selector.js.map +1 -0
- package/dist/cost-routing/task-analyzer.d.ts +13 -0
- package/dist/cost-routing/task-analyzer.d.ts.map +1 -0
- package/dist/cost-routing/task-analyzer.js +185 -0
- package/dist/cost-routing/task-analyzer.js.map +1 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -0
- package/dist/index.js.map +1 -1
- package/dist/learning/ab-testing.d.ts +45 -0
- package/dist/learning/ab-testing.d.ts.map +1 -0
- package/dist/learning/ab-testing.js +267 -0
- package/dist/learning/ab-testing.js.map +1 -0
- package/dist/learning/agent-optimizer.d.ts +42 -0
- package/dist/learning/agent-optimizer.d.ts.map +1 -0
- package/dist/learning/agent-optimizer.js +273 -0
- package/dist/learning/agent-optimizer.js.map +1 -0
- package/dist/learning/auto-optimizer.d.ts +38 -0
- package/dist/learning/auto-optimizer.d.ts.map +1 -0
- package/dist/learning/auto-optimizer.js +229 -0
- package/dist/learning/auto-optimizer.js.map +1 -0
- package/dist/learning/demo-selector.d.ts +29 -0
- package/dist/learning/demo-selector.d.ts.map +1 -0
- package/dist/learning/demo-selector.js +235 -0
- package/dist/learning/demo-selector.js.map +1 -0
- package/dist/learning/index.d.ts +24 -0
- package/dist/learning/index.d.ts.map +1 -0
- package/dist/learning/index.js +13 -0
- package/dist/learning/index.js.map +1 -0
- package/dist/learning/instruction-optimizer.d.ts +29 -0
- package/dist/learning/instruction-optimizer.d.ts.map +1 -0
- package/dist/learning/instruction-optimizer.js +175 -0
- package/dist/learning/instruction-optimizer.js.map +1 -0
- package/dist/learning/metrics.d.ts +37 -0
- package/dist/learning/metrics.d.ts.map +1 -0
- package/dist/learning/metrics.js +310 -0
- package/dist/learning/metrics.js.map +1 -0
- package/dist/learning/postgres-trace-store.d.ts +53 -0
- package/dist/learning/postgres-trace-store.d.ts.map +1 -0
- package/dist/learning/postgres-trace-store.js +692 -0
- package/dist/learning/postgres-trace-store.js.map +1 -0
- package/dist/learning/prompt-logger.d.ts +29 -0
- package/dist/learning/prompt-logger.d.ts.map +1 -0
- package/dist/learning/prompt-logger.js +157 -0
- package/dist/learning/prompt-logger.js.map +1 -0
- package/dist/learning/prompt-monitor.d.ts +29 -0
- package/dist/learning/prompt-monitor.d.ts.map +1 -0
- package/dist/learning/prompt-monitor.js +243 -0
- package/dist/learning/prompt-monitor.js.map +1 -0
- package/dist/learning/prompts.d.ts +28 -0
- package/dist/learning/prompts.d.ts.map +1 -0
- package/dist/learning/prompts.js +195 -0
- package/dist/learning/prompts.js.map +1 -0
- package/dist/learning/rollback-manager.d.ts +36 -0
- package/dist/learning/rollback-manager.d.ts.map +1 -0
- package/dist/learning/rollback-manager.js +177 -0
- package/dist/learning/rollback-manager.js.map +1 -0
- package/dist/learning/trace-store.d.ts +26 -0
- package/dist/learning/trace-store.d.ts.map +1 -0
- package/dist/learning/trace-store.js +218 -0
- package/dist/learning/trace-store.js.map +1 -0
- package/dist/llm/google.d.ts.map +1 -1
- package/dist/llm/google.js +1 -2
- package/dist/llm/google.js.map +1 -1
- package/dist/reasoning/branch-evaluator.d.ts +28 -0
- package/dist/reasoning/branch-evaluator.d.ts.map +1 -0
- package/dist/reasoning/branch-evaluator.js +143 -0
- package/dist/reasoning/branch-evaluator.js.map +1 -0
- package/dist/reasoning/branch-generator.d.ts +9 -0
- package/dist/reasoning/branch-generator.d.ts.map +1 -0
- package/dist/reasoning/branch-generator.js +60 -0
- package/dist/reasoning/branch-generator.js.map +1 -0
- package/dist/reasoning/index.d.ts +5 -0
- package/dist/reasoning/index.d.ts.map +1 -0
- package/dist/reasoning/index.js +5 -0
- package/dist/reasoning/index.js.map +1 -0
- package/dist/reasoning/prompts.d.ts +19 -0
- package/dist/reasoning/prompts.d.ts.map +1 -0
- package/dist/reasoning/prompts.js +161 -0
- package/dist/reasoning/prompts.js.map +1 -0
- package/dist/reasoning/thought-tree.d.ts +32 -0
- package/dist/reasoning/thought-tree.d.ts.map +1 -0
- package/dist/reasoning/thought-tree.js +352 -0
- package/dist/reasoning/thought-tree.js.map +1 -0
- package/dist/reflection/index.d.ts +4 -0
- package/dist/reflection/index.d.ts.map +1 -0
- package/dist/reflection/index.js +4 -0
- package/dist/reflection/index.js.map +1 -0
- package/dist/reflection/insight-store.d.ts +19 -0
- package/dist/reflection/insight-store.d.ts.map +1 -0
- package/dist/reflection/insight-store.js +129 -0
- package/dist/reflection/insight-store.js.map +1 -0
- package/dist/reflection/prompts.d.ts +18 -0
- package/dist/reflection/prompts.d.ts.map +1 -0
- package/dist/reflection/prompts.js +157 -0
- package/dist/reflection/prompts.js.map +1 -0
- package/dist/reflection/reflection-engine.d.ts +25 -0
- package/dist/reflection/reflection-engine.d.ts.map +1 -0
- package/dist/reflection/reflection-engine.js +202 -0
- package/dist/reflection/reflection-engine.js.map +1 -0
- package/dist/registry.d.ts +1 -0
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +3 -0
- package/dist/registry.js.map +1 -1
- package/dist/time-travel/checkpoint-store.d.ts +34 -0
- package/dist/time-travel/checkpoint-store.d.ts.map +1 -0
- package/dist/time-travel/checkpoint-store.js +240 -0
- package/dist/time-travel/checkpoint-store.js.map +1 -0
- package/dist/time-travel/comparator.d.ts +26 -0
- package/dist/time-travel/comparator.d.ts.map +1 -0
- package/dist/time-travel/comparator.js +253 -0
- package/dist/time-travel/comparator.js.map +1 -0
- package/dist/time-travel/forker.d.ts +22 -0
- package/dist/time-travel/forker.d.ts.map +1 -0
- package/dist/time-travel/forker.js +118 -0
- package/dist/time-travel/forker.js.map +1 -0
- package/dist/time-travel/index.d.ts +6 -0
- package/dist/time-travel/index.d.ts.map +1 -0
- package/dist/time-travel/index.js +6 -0
- package/dist/time-travel/index.js.map +1 -0
- package/dist/time-travel/replayer.d.ts +20 -0
- package/dist/time-travel/replayer.d.ts.map +1 -0
- package/dist/time-travel/replayer.js +147 -0
- package/dist/time-travel/replayer.js.map +1 -0
- package/dist/time-travel/time-travel.d.ts +41 -0
- package/dist/time-travel/time-travel.d.ts.map +1 -0
- package/dist/time-travel/time-travel.js +127 -0
- package/dist/time-travel/time-travel.js.map +1 -0
- package/dist/tool.d.ts.map +1 -1
- package/dist/tool.js +2 -0
- package/dist/tool.js.map +1 -1
- package/dist/tools/base64.d.ts.map +1 -1
- package/dist/tools/base64.js +2 -8
- package/dist/tools/base64.js.map +1 -1
- package/dist/tools/datetime.d.ts.map +1 -1
- package/dist/tools/datetime.js.map +1 -1
- package/dist/tools/exec.d.ts.map +1 -1
- package/dist/tools/exec.js +1 -4
- package/dist/tools/exec.js.map +1 -1
- package/dist/tools/filesystem.d.ts.map +1 -1
- package/dist/tools/filesystem.js +4 -1
- package/dist/tools/filesystem.js.map +1 -1
- package/dist/tools/hash.d.ts.map +1 -1
- package/dist/tools/hash.js +1 -4
- package/dist/tools/hash.js.map +1 -1
- package/dist/tools/http.d.ts.map +1 -1
- package/dist/tools/http.js +1 -4
- package/dist/tools/http.js.map +1 -1
- package/dist/tools/regex.d.ts.map +1 -1
- package/dist/tools/regex.js +4 -1
- package/dist/tools/regex.js.map +1 -1
- package/dist/utils/circuit-breaker.d.ts.map +1 -1
- package/dist/utils/circuit-breaker.js.map +1 -1
- package/dist/utils/fallback.d.ts.map +1 -1
- package/dist/utils/fallback.js +1 -4
- package/dist/utils/fallback.js.map +1 -1
- package/dist/utils/retry.d.ts.map +1 -1
- package/dist/utils/retry.js +8 -13
- package/dist/utils/retry.js.map +1 -1
- package/package.json +17 -8
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
export function buildFailureAnalysisPrompt(traces, currentInstructions) {
|
|
2
|
+
const failedTraces = traces.filter((t) => !t.metrics.success || t.score < 0.7);
|
|
3
|
+
const traceSummaries = failedTraces
|
|
4
|
+
.slice(0, 5)
|
|
5
|
+
.map((t, i) => `
|
|
6
|
+
Trace ${i + 1}:
|
|
7
|
+
- Input: ${t.input.slice(0, 200)}${t.input.length > 200 ? '...' : ''}
|
|
8
|
+
- Output: ${t.output.slice(0, 200)}${t.output.length > 200 ? '...' : ''}
|
|
9
|
+
- Score: ${t.score.toFixed(2)}
|
|
10
|
+
- Tools used: ${t.toolCalls.map((tc) => tc.name).join(', ') || 'none'}
|
|
11
|
+
- Error: ${t.steps.find((s) => s.toolResult?.error)?.toolResult?.error ?? 'none'}
|
|
12
|
+
`)
|
|
13
|
+
.join('\n');
|
|
14
|
+
return `Analyze these failed agent executions to identify instruction gaps.
|
|
15
|
+
|
|
16
|
+
Current Instructions:
|
|
17
|
+
${currentInstructions}
|
|
18
|
+
|
|
19
|
+
Failed Executions:
|
|
20
|
+
${traceSummaries}
|
|
21
|
+
|
|
22
|
+
Identify patterns in the failures. What's missing from the instructions that caused these issues?
|
|
23
|
+
|
|
24
|
+
Respond with JSON:
|
|
25
|
+
{
|
|
26
|
+
"gaps": [
|
|
27
|
+
{
|
|
28
|
+
"description": "What the agent should have done differently",
|
|
29
|
+
"frequency": 1-5,
|
|
30
|
+
"suggestedFix": "Specific instruction to add or modify"
|
|
31
|
+
}
|
|
32
|
+
],
|
|
33
|
+
"overallAnalysis": "Brief summary of the main issues"
|
|
34
|
+
}`;
|
|
35
|
+
}
|
|
36
|
+
export function buildInstructionCandidatePrompt(currentInstructions, gaps, insights) {
|
|
37
|
+
const gapsList = gaps
|
|
38
|
+
.map((g, i) => `${i + 1}. ${g.description} (frequency: ${g.frequency})\n Fix: ${g.suggestedFix}`)
|
|
39
|
+
.join('\n');
|
|
40
|
+
const insightsList = insights
|
|
41
|
+
.slice(0, 5)
|
|
42
|
+
.map((ins, i) => `${i + 1}. [${ins.type}] ${ins.content}`)
|
|
43
|
+
.join('\n');
|
|
44
|
+
return `Generate improved agent instructions based on identified gaps and learnings.
|
|
45
|
+
|
|
46
|
+
Current Instructions:
|
|
47
|
+
${currentInstructions}
|
|
48
|
+
|
|
49
|
+
Identified Gaps:
|
|
50
|
+
${gapsList}
|
|
51
|
+
|
|
52
|
+
Past Learnings:
|
|
53
|
+
${insightsList || 'None available'}
|
|
54
|
+
|
|
55
|
+
Generate 3 different improved versions of the instructions. Each should:
|
|
56
|
+
1. Address the identified gaps
|
|
57
|
+
2. Incorporate relevant learnings
|
|
58
|
+
3. Be clear, specific, and actionable
|
|
59
|
+
4. Not be too long (similar length to original)
|
|
60
|
+
|
|
61
|
+
Respond with JSON:
|
|
62
|
+
{
|
|
63
|
+
"candidates": [
|
|
64
|
+
{
|
|
65
|
+
"instructions": "Full improved instructions text",
|
|
66
|
+
"reasoning": "Why this version is better"
|
|
67
|
+
}
|
|
68
|
+
]
|
|
69
|
+
}`;
|
|
70
|
+
}
|
|
71
|
+
export function buildInstructionEvaluationPrompt(candidate, traces) {
|
|
72
|
+
const traceSummaries = traces
|
|
73
|
+
.slice(0, 3)
|
|
74
|
+
.map((t, i) => `
|
|
75
|
+
Example ${i + 1}:
|
|
76
|
+
- Input: ${t.input.slice(0, 150)}
|
|
77
|
+
- Expected behavior: ${t.metrics.success ? 'Success' : 'Should have succeeded'}
|
|
78
|
+
`)
|
|
79
|
+
.join('\n');
|
|
80
|
+
return `Evaluate if these instructions would have helped the agent perform better.
|
|
81
|
+
|
|
82
|
+
Instructions to evaluate:
|
|
83
|
+
${candidate}
|
|
84
|
+
|
|
85
|
+
Example scenarios where the agent struggled:
|
|
86
|
+
${traceSummaries}
|
|
87
|
+
|
|
88
|
+
Would these instructions help the agent handle these scenarios better?
|
|
89
|
+
|
|
90
|
+
Respond with JSON:
|
|
91
|
+
{
|
|
92
|
+
"score": 0.0-1.0,
|
|
93
|
+
"strengths": ["What's good about these instructions"],
|
|
94
|
+
"weaknesses": ["What could still be improved"],
|
|
95
|
+
"reasoning": "Overall assessment"
|
|
96
|
+
}`;
|
|
97
|
+
}
|
|
98
|
+
export function buildInstructionRefinementPrompt(candidate, feedback) {
|
|
99
|
+
return `Refine these agent instructions based on feedback.
|
|
100
|
+
|
|
101
|
+
Current Version:
|
|
102
|
+
${candidate}
|
|
103
|
+
|
|
104
|
+
Feedback:
|
|
105
|
+
${feedback.map((f, i) => `${i + 1}. ${f}`).join('\n')}
|
|
106
|
+
|
|
107
|
+
Create an improved version that addresses the feedback while keeping the good parts.
|
|
108
|
+
|
|
109
|
+
Respond with JSON:
|
|
110
|
+
{
|
|
111
|
+
"instructions": "Refined instructions text",
|
|
112
|
+
"changes": ["List of changes made"]
|
|
113
|
+
}`;
|
|
114
|
+
}
|
|
115
|
+
export function parseFailureAnalysisResponse(content) {
|
|
116
|
+
try {
|
|
117
|
+
let jsonStr = content;
|
|
118
|
+
const codeBlockMatch = /```(?:json)?\s*([\s\S]*?)\s*```/.exec(content);
|
|
119
|
+
if (codeBlockMatch) {
|
|
120
|
+
jsonStr = codeBlockMatch[1];
|
|
121
|
+
}
|
|
122
|
+
const parsed = JSON.parse(jsonStr);
|
|
123
|
+
const gaps = (parsed.gaps ?? [])
|
|
124
|
+
.filter((g) => g && typeof g === 'object' && 'description' in g)
|
|
125
|
+
.map((g) => ({
|
|
126
|
+
description: String(g.description ?? ''),
|
|
127
|
+
frequency: Math.min(5, Math.max(1, Number(g.frequency) || 1)),
|
|
128
|
+
exampleTraces: [],
|
|
129
|
+
suggestedFix: String(g.suggestedFix ?? ''),
|
|
130
|
+
}));
|
|
131
|
+
return {
|
|
132
|
+
gaps,
|
|
133
|
+
overallAnalysis: String(parsed.overallAnalysis ?? ''),
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
export function parseInstructionCandidatesResponse(content) {
|
|
141
|
+
try {
|
|
142
|
+
let jsonStr = content;
|
|
143
|
+
const codeBlockMatch = /```(?:json)?\s*([\s\S]*?)\s*```/.exec(content);
|
|
144
|
+
if (codeBlockMatch) {
|
|
145
|
+
jsonStr = codeBlockMatch[1];
|
|
146
|
+
}
|
|
147
|
+
const parsed = JSON.parse(jsonStr);
|
|
148
|
+
return (parsed.candidates ?? [])
|
|
149
|
+
.filter((c) => c && typeof c === 'object' && 'instructions' in c)
|
|
150
|
+
.map((c) => ({
|
|
151
|
+
instructions: String(c.instructions ?? ''),
|
|
152
|
+
reasoning: String(c.reasoning ?? ''),
|
|
153
|
+
}));
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
return [];
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
export function parseInstructionEvaluationResponse(content) {
|
|
160
|
+
try {
|
|
161
|
+
let jsonStr = content;
|
|
162
|
+
const codeBlockMatch = /```(?:json)?\s*([\s\S]*?)\s*```/.exec(content);
|
|
163
|
+
if (codeBlockMatch) {
|
|
164
|
+
jsonStr = codeBlockMatch[1];
|
|
165
|
+
}
|
|
166
|
+
const parsed = JSON.parse(jsonStr);
|
|
167
|
+
return {
|
|
168
|
+
score: Math.max(0, Math.min(1, Number(parsed.score) || 0.5)),
|
|
169
|
+
strengths: Array.isArray(parsed.strengths) ? parsed.strengths.map(String) : [],
|
|
170
|
+
weaknesses: Array.isArray(parsed.weaknesses) ? parsed.weaknesses.map(String) : [],
|
|
171
|
+
reasoning: String(parsed.reasoning ?? ''),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
export function parseInstructionRefinementResponse(content) {
|
|
179
|
+
try {
|
|
180
|
+
let jsonStr = content;
|
|
181
|
+
const codeBlockMatch = /```(?:json)?\s*([\s\S]*?)\s*```/.exec(content);
|
|
182
|
+
if (codeBlockMatch) {
|
|
183
|
+
jsonStr = codeBlockMatch[1];
|
|
184
|
+
}
|
|
185
|
+
const parsed = JSON.parse(jsonStr);
|
|
186
|
+
return {
|
|
187
|
+
instructions: String(parsed.instructions ?? ''),
|
|
188
|
+
changes: Array.isArray(parsed.changes) ? parsed.changes.map(String) : [],
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
//# sourceMappingURL=prompts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../src/learning/prompts.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,0BAA0B,CACxC,MAAwB,EACxB,mBAA2B;IAE3B,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;IAE/E,MAAM,cAAc,GAAG,YAAY;SAChC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CACF,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACR,CAAC,GAAG,CAAC;WACF,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YACxD,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;WAC5D,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBACb,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM;WAC1D,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,UAAU,EAAE,KAAK,IAAI,MAAM;CAC/E,CACI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;;;EAGP,mBAAmB;;;EAGnB,cAAc;;;;;;;;;;;;;;EAcd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,mBAA2B,EAC3B,IAAsB,EACtB,QAAmB;IAEnB,MAAM,QAAQ,GAAG,IAAI;SAClB,GAAG,CACF,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,WAAW,gBAAgB,CAAC,CAAC,SAAS,cAAc,CAAC,CAAC,YAAY,EAAE,CAC9F;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,YAAY,GAAG,QAAQ;SAC1B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC;SACzD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;;;EAGP,mBAAmB;;;EAGnB,QAAQ;;;EAGR,YAAY,IAAI,gBAAgB;;;;;;;;;;;;;;;;EAgBhC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gCAAgC,CAC9C,SAAiB,EACjB,MAAwB;IAExB,MAAM,cAAc,GAAG,MAAM;SAC1B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CACF,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;UACN,CAAC,GAAG,CAAC;WACJ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;uBACT,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,uBAAuB;CAC7E,CACI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;;;EAGP,SAAS;;;EAGT,cAAc;;;;;;;;;;EAUd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gCAAgC,CAAC,SAAiB,EAAE,QAAkB;IACpF,OAAO;;;EAGP,SAAS;;;EAGT,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;EAQnD,CAAC;AACH,CAAC;AAOD,MAAM,UAAU,4BAA4B,CAAC,OAAe;IAC1D,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,OAAO,CAAC;QACtB,MAAM,cAAc,GAAG,iCAAiC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvE,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,MAAM,IAAI,GAAqB,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;aAC/C,MAAM,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,aAAa,IAAI,CAAC,CAAC;aACxE,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE,CAAC,CAAC;YACpC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC;YACxC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;YAC7D,aAAa,EAAE,EAAE;YACjB,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC;SAC3C,CAAC,CAAC,CAAC;QAEN,OAAO;YACL,IAAI;YACJ,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAOD,MAAM,UAAU,kCAAkC,CAAC,OAAe;IAChE,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,OAAO,CAAC;QACtB,MAAM,cAAc,GAAG,iCAAiC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvE,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;aAC7B,MAAM,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,cAAc,IAAI,CAAC,CAAC;aACzE,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE,CAAC,CAAC;YACpC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC;YAC1C,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;SACrC,CAAC,CAAC,CAAC;IACR,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AASD,MAAM,UAAU,kCAAkC,CAAC,OAAe;IAChE,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,OAAO,CAAC;QACtB,MAAM,cAAc,GAAG,iCAAiC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvE,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;YAC5D,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;YAC9E,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;YACjF,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;SAC1C,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAOD,MAAM,UAAU,kCAAkC,CAAC,OAAe;IAChE,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,OAAO,CAAC;QACtB,MAAM,cAAc,GAAG,iCAAiC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvE,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,OAAO;YACL,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC;YAC/C,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;SACzE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { InstructionVersion, InstructionVersionStore, InstructionSource } from '@cogitator-ai/types';
|
|
2
|
+
export interface RollbackManagerConfig {
|
|
3
|
+
store: InstructionVersionStore;
|
|
4
|
+
maxVersionsToKeep?: number;
|
|
5
|
+
}
|
|
6
|
+
export interface RollbackResult {
|
|
7
|
+
success: boolean;
|
|
8
|
+
previousVersion: InstructionVersion;
|
|
9
|
+
newVersion: InstructionVersion;
|
|
10
|
+
message: string;
|
|
11
|
+
}
|
|
12
|
+
export declare class RollbackManager {
|
|
13
|
+
private store;
|
|
14
|
+
private maxVersionsToKeep;
|
|
15
|
+
constructor(config: RollbackManagerConfig);
|
|
16
|
+
getCurrentVersion(agentId: string): Promise<InstructionVersion | null>;
|
|
17
|
+
getVersionHistory(agentId: string, limit?: number): Promise<InstructionVersion[]>;
|
|
18
|
+
getVersion(versionId: string): Promise<InstructionVersion | null>;
|
|
19
|
+
deployVersion(agentId: string, instructions: string, source: InstructionSource, sourceId?: string): Promise<InstructionVersion>;
|
|
20
|
+
rollbackTo(agentId: string, targetVersionId: string): Promise<RollbackResult>;
|
|
21
|
+
rollbackToPrevious(agentId: string): Promise<RollbackResult>;
|
|
22
|
+
recordMetrics(agentId: string, score: number, latency: number, cost: number, success: boolean): Promise<void>;
|
|
23
|
+
compareVersions(versionId1: string, versionId2: string): Promise<{
|
|
24
|
+
v1: InstructionVersion;
|
|
25
|
+
v2: InstructionVersion;
|
|
26
|
+
comparison: {
|
|
27
|
+
scoreDiff: number;
|
|
28
|
+
latencyDiff: number;
|
|
29
|
+
successRateDiff: number;
|
|
30
|
+
recommendation: string;
|
|
31
|
+
};
|
|
32
|
+
} | null>;
|
|
33
|
+
findBestVersion(agentId: string): Promise<InstructionVersion | null>;
|
|
34
|
+
private pruneOldVersions;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=rollback-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rollback-manager.d.ts","sourceRoot":"","sources":["../../src/learning/rollback-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAClB,uBAAuB,EACvB,iBAAiB,EAClB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,uBAAuB,CAAC;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,EAAE,kBAAkB,CAAC;IACpC,UAAU,EAAE,kBAAkB,CAAC;IAC/B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,KAAK,CAA0B;IACvC,OAAO,CAAC,iBAAiB,CAAS;gBAEtB,MAAM,EAAE,qBAAqB;IAKnC,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAItE,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAIjF,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAIjE,aAAa,CACjB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,iBAAiB,EACzB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,kBAAkB,CAAC;IA+BxB,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IA8C7E,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAgB5D,aAAa,CACjB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,IAAI,CAAC;IAyBV,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QACrE,EAAE,EAAE,kBAAkB,CAAC;QACvB,EAAE,EAAE,kBAAkB,CAAC;QACvB,UAAU,EAAE;YACV,SAAS,EAAE,MAAM,CAAC;YAClB,WAAW,EAAE,MAAM,CAAC;YACpB,eAAe,EAAE,MAAM,CAAC;YACxB,cAAc,EAAE,MAAM,CAAC;SACxB,CAAC;KACH,GAAG,IAAI,CAAC;IAwCH,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;YAoB5D,gBAAgB;CAc/B"}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
export class RollbackManager {
|
|
2
|
+
store;
|
|
3
|
+
maxVersionsToKeep;
|
|
4
|
+
constructor(config) {
|
|
5
|
+
this.store = config.store;
|
|
6
|
+
this.maxVersionsToKeep = config.maxVersionsToKeep ?? 20;
|
|
7
|
+
}
|
|
8
|
+
async getCurrentVersion(agentId) {
|
|
9
|
+
return this.store.getCurrent(agentId);
|
|
10
|
+
}
|
|
11
|
+
async getVersionHistory(agentId, limit) {
|
|
12
|
+
return this.store.getHistory(agentId, limit ?? this.maxVersionsToKeep);
|
|
13
|
+
}
|
|
14
|
+
async getVersion(versionId) {
|
|
15
|
+
return this.store.get(versionId);
|
|
16
|
+
}
|
|
17
|
+
async deployVersion(agentId, instructions, source, sourceId) {
|
|
18
|
+
const current = await this.store.getCurrent(agentId);
|
|
19
|
+
if (current) {
|
|
20
|
+
await this.store.retire(current.id);
|
|
21
|
+
}
|
|
22
|
+
const nextVersion = current ? current.version + 1 : 1;
|
|
23
|
+
const newVersion = await this.store.save({
|
|
24
|
+
agentId,
|
|
25
|
+
version: nextVersion,
|
|
26
|
+
instructions,
|
|
27
|
+
source,
|
|
28
|
+
sourceId,
|
|
29
|
+
deployedAt: new Date(),
|
|
30
|
+
metrics: {
|
|
31
|
+
runCount: 0,
|
|
32
|
+
avgScore: 0,
|
|
33
|
+
successRate: 0,
|
|
34
|
+
avgLatency: 0,
|
|
35
|
+
totalCost: 0,
|
|
36
|
+
},
|
|
37
|
+
parentVersionId: current?.id,
|
|
38
|
+
});
|
|
39
|
+
await this.pruneOldVersions(agentId);
|
|
40
|
+
return newVersion;
|
|
41
|
+
}
|
|
42
|
+
async rollbackTo(agentId, targetVersionId) {
|
|
43
|
+
const current = await this.store.getCurrent(agentId);
|
|
44
|
+
const targetVersion = await this.store.get(targetVersionId);
|
|
45
|
+
if (!current) {
|
|
46
|
+
return {
|
|
47
|
+
success: false,
|
|
48
|
+
previousVersion: null,
|
|
49
|
+
newVersion: null,
|
|
50
|
+
message: 'No current version found',
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
if (!targetVersion) {
|
|
54
|
+
return {
|
|
55
|
+
success: false,
|
|
56
|
+
previousVersion: current,
|
|
57
|
+
newVersion: null,
|
|
58
|
+
message: `Target version ${targetVersionId} not found`,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
if (targetVersion.agentId !== agentId) {
|
|
62
|
+
return {
|
|
63
|
+
success: false,
|
|
64
|
+
previousVersion: current,
|
|
65
|
+
newVersion: null,
|
|
66
|
+
message: 'Target version belongs to a different agent',
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
const newVersion = await this.deployVersion(agentId, targetVersion.instructions, 'rollback', targetVersion.id);
|
|
70
|
+
return {
|
|
71
|
+
success: true,
|
|
72
|
+
previousVersion: current,
|
|
73
|
+
newVersion,
|
|
74
|
+
message: `Successfully rolled back from v${current.version} to v${newVersion.version} (based on v${targetVersion.version})`,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
async rollbackToPrevious(agentId) {
|
|
78
|
+
const history = await this.store.getHistory(agentId, 2);
|
|
79
|
+
if (history.length < 2) {
|
|
80
|
+
return {
|
|
81
|
+
success: false,
|
|
82
|
+
previousVersion: history[0] || null,
|
|
83
|
+
newVersion: null,
|
|
84
|
+
message: 'No previous version available for rollback',
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
const previous = history[1];
|
|
88
|
+
return this.rollbackTo(agentId, previous.id);
|
|
89
|
+
}
|
|
90
|
+
async recordMetrics(agentId, score, latency, cost, success) {
|
|
91
|
+
const current = await this.store.getCurrent(agentId);
|
|
92
|
+
if (!current)
|
|
93
|
+
return;
|
|
94
|
+
const runCount = current.metrics.runCount + 1;
|
|
95
|
+
const successCount = current.metrics.successRate * current.metrics.runCount + (success ? 1 : 0);
|
|
96
|
+
const newSuccessRate = successCount / runCount;
|
|
97
|
+
const totalScore = current.metrics.avgScore * current.metrics.runCount + score;
|
|
98
|
+
const newAvgScore = totalScore / runCount;
|
|
99
|
+
const totalLatency = current.metrics.avgLatency * current.metrics.runCount + latency;
|
|
100
|
+
const newAvgLatency = totalLatency / runCount;
|
|
101
|
+
const newTotalCost = current.metrics.totalCost + cost;
|
|
102
|
+
await this.store.updateMetrics(current.id, {
|
|
103
|
+
runCount,
|
|
104
|
+
avgScore: newAvgScore,
|
|
105
|
+
successRate: newSuccessRate,
|
|
106
|
+
avgLatency: newAvgLatency,
|
|
107
|
+
totalCost: newTotalCost,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
async compareVersions(versionId1, versionId2) {
|
|
111
|
+
const v1 = await this.store.get(versionId1);
|
|
112
|
+
const v2 = await this.store.get(versionId2);
|
|
113
|
+
if (!v1 || !v2) {
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
const scoreDiff = v2.metrics.avgScore - v1.metrics.avgScore;
|
|
117
|
+
const latencyDiff = v2.metrics.avgLatency - v1.metrics.avgLatency;
|
|
118
|
+
const successRateDiff = v2.metrics.successRate - v1.metrics.successRate;
|
|
119
|
+
let recommendation;
|
|
120
|
+
if (v1.metrics.runCount < 10 || v2.metrics.runCount < 10) {
|
|
121
|
+
recommendation = 'Insufficient data for meaningful comparison';
|
|
122
|
+
}
|
|
123
|
+
else if (scoreDiff > 0.05 && successRateDiff >= 0) {
|
|
124
|
+
recommendation = `v${v2.version} performs better with ${(scoreDiff * 100).toFixed(1)}% higher score`;
|
|
125
|
+
}
|
|
126
|
+
else if (scoreDiff < -0.05 && successRateDiff <= 0) {
|
|
127
|
+
recommendation = `v${v1.version} performs better with ${(-scoreDiff * 100).toFixed(1)}% higher score`;
|
|
128
|
+
}
|
|
129
|
+
else if (latencyDiff < 0 && Math.abs(scoreDiff) < 0.02) {
|
|
130
|
+
recommendation = `v${v2.version} is faster with similar quality`;
|
|
131
|
+
}
|
|
132
|
+
else if (latencyDiff > 0 && Math.abs(scoreDiff) < 0.02) {
|
|
133
|
+
recommendation = `v${v1.version} is faster with similar quality`;
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
recommendation = 'Performance is comparable between versions';
|
|
137
|
+
}
|
|
138
|
+
return {
|
|
139
|
+
v1,
|
|
140
|
+
v2,
|
|
141
|
+
comparison: {
|
|
142
|
+
scoreDiff,
|
|
143
|
+
latencyDiff,
|
|
144
|
+
successRateDiff,
|
|
145
|
+
recommendation,
|
|
146
|
+
},
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
async findBestVersion(agentId) {
|
|
150
|
+
const history = await this.store.getHistory(agentId, this.maxVersionsToKeep);
|
|
151
|
+
if (history.length === 0) {
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
const versionsWithEnoughData = history.filter((v) => v.metrics.runCount >= 10);
|
|
155
|
+
if (versionsWithEnoughData.length === 0) {
|
|
156
|
+
return history[0];
|
|
157
|
+
}
|
|
158
|
+
return versionsWithEnoughData.reduce((best, current) => {
|
|
159
|
+
const currentWeightedScore = current.metrics.avgScore * current.metrics.successRate;
|
|
160
|
+
const bestWeightedScore = best.metrics.avgScore * best.metrics.successRate;
|
|
161
|
+
return currentWeightedScore > bestWeightedScore ? current : best;
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
async pruneOldVersions(agentId) {
|
|
165
|
+
const history = await this.store.getHistory(agentId, this.maxVersionsToKeep + 10);
|
|
166
|
+
if (history.length <= this.maxVersionsToKeep) {
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
const toRetire = history.slice(this.maxVersionsToKeep);
|
|
170
|
+
for (const version of toRetire) {
|
|
171
|
+
if (!version.retiredAt) {
|
|
172
|
+
await this.store.retire(version.id);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
//# sourceMappingURL=rollback-manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rollback-manager.js","sourceRoot":"","sources":["../../src/learning/rollback-manager.ts"],"names":[],"mappings":"AAkBA,MAAM,OAAO,eAAe;IAClB,KAAK,CAA0B;IAC/B,iBAAiB,CAAS;IAElC,YAAY,MAA6B;QACvC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,IAAI,EAAE,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAAe;QACrC,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAAe,EAAE,KAAc;QACrD,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,SAAiB;QAChC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,OAAe,EACf,YAAoB,EACpB,MAAyB,EACzB,QAAiB;QAEjB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAErD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACvC,OAAO;YACP,OAAO,EAAE,WAAW;YACpB,YAAY;YACZ,MAAM;YACN,QAAQ;YACR,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,OAAO,EAAE;gBACP,QAAQ,EAAE,CAAC;gBACX,QAAQ,EAAE,CAAC;gBACX,WAAW,EAAE,CAAC;gBACd,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,CAAC;aACb;YACD,eAAe,EAAE,OAAO,EAAE,EAAE;SAC7B,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAErC,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAe,EAAE,eAAuB;QACvD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAE5D,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,eAAe,EAAE,IAAqC;gBACtD,UAAU,EAAE,IAAqC;gBACjD,OAAO,EAAE,0BAA0B;aACpC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,eAAe,EAAE,OAAO;gBACxB,UAAU,EAAE,IAAqC;gBACjD,OAAO,EAAE,kBAAkB,eAAe,YAAY;aACvD,CAAC;QACJ,CAAC;QAED,IAAI,aAAa,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YACtC,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,eAAe,EAAE,OAAO;gBACxB,UAAU,EAAE,IAAqC;gBACjD,OAAO,EAAE,6CAA6C;aACvD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CACzC,OAAO,EACP,aAAa,CAAC,YAAY,EAC1B,UAAU,EACV,aAAa,CAAC,EAAE,CACjB,CAAC;QAEF,OAAO;YACL,OAAO,EAAE,IAAI;YACb,eAAe,EAAE,OAAO;YACxB,UAAU;YACV,OAAO,EAAE,kCAAkC,OAAO,CAAC,OAAO,QAAQ,UAAU,CAAC,OAAO,eAAe,aAAa,CAAC,OAAO,GAAG;SAC5H,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAe;QACtC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAExD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC,IAAK,IAAsC;gBACtE,UAAU,EAAE,IAAqC;gBACjD,OAAO,EAAE,4CAA4C;aACtD,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,OAAe,EACf,KAAa,EACb,OAAe,EACf,IAAY,EACZ,OAAgB;QAEhB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC9C,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChG,MAAM,cAAc,GAAG,YAAY,GAAG,QAAQ,CAAC;QAE/C,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;QAC/E,MAAM,WAAW,GAAG,UAAU,GAAG,QAAQ,CAAC;QAE1C,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC;QACrF,MAAM,aAAa,GAAG,YAAY,GAAG,QAAQ,CAAC;QAE9C,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;YACzC,QAAQ;YACR,QAAQ,EAAE,WAAW;YACrB,WAAW,EAAE,cAAc;YAC3B,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,YAAY;SACxB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,UAAkB,EAAE,UAAkB;QAU1D,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE5C,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC5D,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC;QAClE,MAAM,eAAe,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC;QAExE,IAAI,cAAsB,CAAC;QAE3B,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,GAAG,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,GAAG,EAAE,EAAE,CAAC;YACzD,cAAc,GAAG,6CAA6C,CAAC;QACjE,CAAC;aAAM,IAAI,SAAS,GAAG,IAAI,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;YACpD,cAAc,GAAG,IAAI,EAAE,CAAC,OAAO,yBAAyB,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC;QACvG,CAAC;aAAM,IAAI,SAAS,GAAG,CAAC,IAAI,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;YACrD,cAAc,GAAG,IAAI,EAAE,CAAC,OAAO,yBAAyB,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC;QACxG,CAAC;aAAM,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,IAAI,EAAE,CAAC;YACzD,cAAc,GAAG,IAAI,EAAE,CAAC,OAAO,iCAAiC,CAAC;QACnE,CAAC;aAAM,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,IAAI,EAAE,CAAC;YACzD,cAAc,GAAG,IAAI,EAAE,CAAC,OAAO,iCAAiC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,4CAA4C,CAAC;QAChE,CAAC;QAED,OAAO;YACL,EAAE;YACF,EAAE;YACF,UAAU,EAAE;gBACV,SAAS;gBACT,WAAW;gBACX,eAAe;gBACf,cAAc;aACf;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe;QACnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAE7E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAE/E,IAAI,sBAAsB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxC,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QAED,OAAO,sBAAsB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YACrD,MAAM,oBAAoB,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;YACpF,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YAC3E,OAAO,oBAAoB,GAAG,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,OAAe;QAC5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,CAAC;QAElF,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC7C,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;gBACvB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { ExecutionTrace, TraceStore, TraceQuery, TraceStoreStats } from '@cogitator-ai/types';
|
|
2
|
+
export declare class InMemoryTraceStore implements TraceStore {
|
|
3
|
+
private traces;
|
|
4
|
+
private agentIndex;
|
|
5
|
+
private demoIndex;
|
|
6
|
+
private runIdIndex;
|
|
7
|
+
store(trace: ExecutionTrace): Promise<void>;
|
|
8
|
+
storeMany(traces: ExecutionTrace[]): Promise<void>;
|
|
9
|
+
get(id: string): Promise<ExecutionTrace | null>;
|
|
10
|
+
getByRunId(runId: string): Promise<ExecutionTrace | null>;
|
|
11
|
+
query(query: TraceQuery): Promise<ExecutionTrace[]>;
|
|
12
|
+
getAll(agentId: string): Promise<ExecutionTrace[]>;
|
|
13
|
+
getDemos(agentId: string, limit?: number): Promise<ExecutionTrace[]>;
|
|
14
|
+
markAsDemo(id: string): Promise<void>;
|
|
15
|
+
unmarkAsDemo(id: string): Promise<void>;
|
|
16
|
+
delete(id: string): Promise<boolean>;
|
|
17
|
+
prune(agentId: string, maxTraces: number): Promise<number>;
|
|
18
|
+
clear(agentId: string): Promise<void>;
|
|
19
|
+
getStats(agentId: string): Promise<TraceStoreStats>;
|
|
20
|
+
getGlobalStats(): {
|
|
21
|
+
totalTraces: number;
|
|
22
|
+
agentCount: number;
|
|
23
|
+
demoCount: number;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=trace-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"trace-store.d.ts","sourceRoot":"","sources":["../../src/learning/trace-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEnG,qBAAa,kBAAmB,YAAW,UAAU;IACnD,OAAO,CAAC,MAAM,CAAqC;IACnD,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,UAAU,CAA6B;IAEzC,KAAK,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB3C,SAAS,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlD,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAI/C,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAMzD,KAAK,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IA6CnD,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAclD,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAchE,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcrC,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQvC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWpC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA6B1D,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBrC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IA6CzD,cAAc,IAAI;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;CAYjF"}
|