@cogitator-ai/self-modifying 17.0.18 → 19.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -50
- package/dist/architecture-evolution/capability-analyzer.d.ts.map +1 -1
- package/dist/architecture-evolution/capability-analyzer.js +38 -37
- package/dist/architecture-evolution/capability-analyzer.js.map +1 -1
- package/dist/architecture-evolution/index.d.ts +2 -2
- package/dist/architecture-evolution/index.d.ts.map +1 -1
- package/dist/architecture-evolution/index.js +1 -1
- package/dist/architecture-evolution/index.js.map +1 -1
- package/dist/architecture-evolution/parameter-optimizer.d.ts +5 -4
- package/dist/architecture-evolution/parameter-optimizer.d.ts.map +1 -1
- package/dist/architecture-evolution/parameter-optimizer.js +36 -31
- package/dist/architecture-evolution/parameter-optimizer.js.map +1 -1
- package/dist/architecture-evolution/prompts.d.ts +21 -2
- package/dist/architecture-evolution/prompts.d.ts.map +1 -1
- package/dist/architecture-evolution/prompts.js +150 -36
- package/dist/architecture-evolution/prompts.js.map +1 -1
- package/dist/constraints/modification-validator.d.ts +1 -0
- package/dist/constraints/modification-validator.d.ts.map +1 -1
- package/dist/constraints/modification-validator.js +30 -14
- package/dist/constraints/modification-validator.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/meta-reasoning/meta-reasoner.d.ts +3 -0
- package/dist/meta-reasoning/meta-reasoner.d.ts.map +1 -1
- package/dist/meta-reasoning/meta-reasoner.js +101 -33
- package/dist/meta-reasoning/meta-reasoner.js.map +1 -1
- package/dist/meta-reasoning/prompts.d.ts +10 -14
- package/dist/meta-reasoning/prompts.d.ts.map +1 -1
- package/dist/meta-reasoning/prompts.js +109 -1
- package/dist/meta-reasoning/prompts.js.map +1 -1
- package/dist/self-modifying-agent.d.ts +83 -17
- package/dist/self-modifying-agent.d.ts.map +1 -1
- package/dist/self-modifying-agent.js +557 -444
- package/dist/self-modifying-agent.js.map +1 -1
- package/dist/tool-generation/gap-analyzer.d.ts +0 -1
- package/dist/tool-generation/gap-analyzer.d.ts.map +1 -1
- package/dist/tool-generation/gap-analyzer.js +14 -14
- package/dist/tool-generation/gap-analyzer.js.map +1 -1
- package/dist/tool-generation/index.d.ts +3 -3
- package/dist/tool-generation/index.d.ts.map +1 -1
- package/dist/tool-generation/index.js +2 -2
- package/dist/tool-generation/index.js.map +1 -1
- package/dist/tool-generation/prompts.d.ts +1 -0
- package/dist/tool-generation/prompts.d.ts.map +1 -1
- package/dist/tool-generation/prompts.js +33 -8
- package/dist/tool-generation/prompts.js.map +1 -1
- package/dist/tool-generation/tool-generator.d.ts +5 -5
- package/dist/tool-generation/tool-generator.d.ts.map +1 -1
- package/dist/tool-generation/tool-generator.js +41 -28
- package/dist/tool-generation/tool-generator.js.map +1 -1
- package/dist/tool-generation/tool-sandbox.d.ts +9 -6
- package/dist/tool-generation/tool-sandbox.d.ts.map +1 -1
- package/dist/tool-generation/tool-sandbox.js +257 -157
- package/dist/tool-generation/tool-sandbox.js.map +1 -1
- package/dist/tool-generation/tool-validator.d.ts +3 -8
- package/dist/tool-generation/tool-validator.d.ts.map +1 -1
- package/dist/tool-generation/tool-validator.js +53 -54
- package/dist/tool-generation/tool-validator.js.map +1 -1
- package/dist/utils/llm-helper.d.ts.map +1 -1
- package/dist/utils/llm-helper.js +1 -0
- package/dist/utils/llm-helper.js.map +1 -1
- package/package.json +11 -18
- package/src/__tests__/component-regressions.test.ts +599 -0
- package/src/__tests__/meta-reasoning.test.ts +5 -1
- package/src/__tests__/sandbox-security.test.ts +186 -0
- package/src/__tests__/self-modifying-agent.test.ts +363 -1
- package/src/architecture-evolution/capability-analyzer.ts +56 -40
- package/src/architecture-evolution/index.ts +2 -0
- package/src/architecture-evolution/parameter-optimizer.ts +52 -34
- package/src/architecture-evolution/prompts.ts +179 -41
- package/src/constraints/modification-validator.ts +30 -14
- package/src/index.ts +14 -0
- package/src/meta-reasoning/meta-reasoner.ts +105 -40
- package/src/meta-reasoning/prompts.ts +129 -15
- package/src/self-modifying-agent.ts +764 -489
- package/src/tool-generation/gap-analyzer.ts +19 -18
- package/src/tool-generation/index.ts +13 -3
- package/src/tool-generation/prompts.ts +41 -8
- package/src/tool-generation/tool-generator.ts +59 -33
- package/src/tool-generation/tool-sandbox.ts +306 -175
- package/src/tool-generation/tool-validator.ts +66 -63
- package/src/utils/llm-helper.ts +1 -0
|
@@ -1,5 +1,21 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
1
2
|
import type { Tool, LLMBackend, TaskProfile } from '@cogitator-ai/types';
|
|
2
3
|
import { buildTaskProfilePrompt, parseTaskProfileResponse } from './prompts';
|
|
4
|
+
import { llmChat } from '../utils/llm-helper';
|
|
5
|
+
|
|
6
|
+
function escapeRegExp(value: string): string {
|
|
7
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function containsPhrase(text: string, phrase: string): boolean {
|
|
11
|
+
return new RegExp(`(?:^|[^a-z0-9])${escapeRegExp(phrase)}(?:s|es|ed|ing)?(?:$|[^a-z0-9])`).test(
|
|
12
|
+
text
|
|
13
|
+
);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function tokenize(text: string): string[] {
|
|
17
|
+
return text.split(/[^a-z0-9-]+/).filter(Boolean);
|
|
18
|
+
}
|
|
3
19
|
|
|
4
20
|
export interface CapabilityAnalyzerOptions {
|
|
5
21
|
llm?: LLMBackend;
|
|
@@ -146,7 +162,7 @@ export class CapabilityAnalyzer {
|
|
|
146
162
|
constraints?: { maxCost?: number; maxLatency?: number };
|
|
147
163
|
}
|
|
148
164
|
): Promise<TaskProfile> {
|
|
149
|
-
const cacheKey = this.buildCacheKey(taskDescription);
|
|
165
|
+
const cacheKey = this.buildCacheKey(taskDescription, context);
|
|
150
166
|
const cached = this.profileCache.get(cacheKey);
|
|
151
167
|
if (cached && Date.now() - cached.timestamp < this.cacheTTL) {
|
|
152
168
|
return cached.profile;
|
|
@@ -190,32 +206,20 @@ export class CapabilityAnalyzer {
|
|
|
190
206
|
constraints: context?.constraints,
|
|
191
207
|
});
|
|
192
208
|
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
messages: [
|
|
208
|
-
{
|
|
209
|
-
role: 'system',
|
|
210
|
-
content:
|
|
211
|
-
'You are a task analysis expert. Analyze tasks and determine their characteristics.',
|
|
212
|
-
},
|
|
213
|
-
{ role: 'user', content: prompt },
|
|
214
|
-
],
|
|
215
|
-
temperature: 0.2,
|
|
216
|
-
});
|
|
217
|
-
|
|
218
|
-
const parsed = parseTaskProfileResponse(response.content);
|
|
209
|
+
const content = await llmChat(
|
|
210
|
+
llm,
|
|
211
|
+
[
|
|
212
|
+
{
|
|
213
|
+
role: 'system',
|
|
214
|
+
content:
|
|
215
|
+
'You are a task analysis expert. Analyze tasks and determine their characteristics.',
|
|
216
|
+
},
|
|
217
|
+
{ role: 'user', content: prompt },
|
|
218
|
+
],
|
|
219
|
+
{ model: this.model, temperature: 0.2 }
|
|
220
|
+
);
|
|
221
|
+
|
|
222
|
+
const parsed = parseTaskProfileResponse(content);
|
|
219
223
|
if (parsed) {
|
|
220
224
|
return parsed;
|
|
221
225
|
}
|
|
@@ -226,8 +230,7 @@ export class CapabilityAnalyzer {
|
|
|
226
230
|
|
|
227
231
|
private analyzeHeuristically(taskDescription: string, availableTools?: Tool[]): TaskProfile {
|
|
228
232
|
const lowerTask = taskDescription.toLowerCase();
|
|
229
|
-
const
|
|
230
|
-
const wordCount = words.length;
|
|
233
|
+
const wordCount = tokenize(lowerTask).length;
|
|
231
234
|
|
|
232
235
|
const domain = this.detectDomain(lowerTask);
|
|
233
236
|
const complexity = this.detectComplexity(lowerTask, wordCount);
|
|
@@ -260,7 +263,7 @@ export class CapabilityAnalyzer {
|
|
|
260
263
|
|
|
261
264
|
for (const [domain, keywords] of Object.entries(DOMAIN_KEYWORDS)) {
|
|
262
265
|
for (const keyword of keywords) {
|
|
263
|
-
if (task
|
|
266
|
+
if (containsPhrase(task, keyword)) {
|
|
264
267
|
scores[domain] += keyword.length > 5 ? 2 : 1;
|
|
265
268
|
}
|
|
266
269
|
}
|
|
@@ -272,7 +275,7 @@ export class CapabilityAnalyzer {
|
|
|
272
275
|
}
|
|
273
276
|
|
|
274
277
|
private detectComplexity(task: string, wordCount: number): TaskProfile['complexity'] {
|
|
275
|
-
const taskWords = new Set(task
|
|
278
|
+
const taskWords = new Set(tokenize(task));
|
|
276
279
|
|
|
277
280
|
for (const [level, indicators] of Object.entries(COMPLEXITY_INDICATORS).reverse()) {
|
|
278
281
|
for (const indicator of indicators) {
|
|
@@ -309,14 +312,14 @@ export class CapabilityAnalyzer {
|
|
|
309
312
|
|
|
310
313
|
let toolMentions = 0;
|
|
311
314
|
for (const indicator of toolIndicators) {
|
|
312
|
-
if (task
|
|
315
|
+
if (containsPhrase(task, indicator)) {
|
|
313
316
|
toolMentions++;
|
|
314
317
|
}
|
|
315
318
|
}
|
|
316
319
|
|
|
317
320
|
if (availableTools?.length) {
|
|
318
321
|
for (const tool of availableTools) {
|
|
319
|
-
if (task
|
|
322
|
+
if (containsPhrase(task, tool.name.toLowerCase())) {
|
|
320
323
|
toolMentions += 2;
|
|
321
324
|
}
|
|
322
325
|
}
|
|
@@ -380,7 +383,7 @@ export class CapabilityAnalyzer {
|
|
|
380
383
|
): TaskProfile['accuracyRequirement'] {
|
|
381
384
|
const criticalIndicators = ['critical', 'exact', 'precise', 'must be correct', 'no errors'];
|
|
382
385
|
for (const indicator of criticalIndicators) {
|
|
383
|
-
if (task
|
|
386
|
+
if (containsPhrase(task, indicator)) {
|
|
384
387
|
return 'critical';
|
|
385
388
|
}
|
|
386
389
|
}
|
|
@@ -391,7 +394,7 @@ export class CapabilityAnalyzer {
|
|
|
391
394
|
|
|
392
395
|
const approximateIndicators = ['rough', 'approximate', 'estimate', 'about', 'roughly'];
|
|
393
396
|
for (const indicator of approximateIndicators) {
|
|
394
|
-
if (task
|
|
397
|
+
if (containsPhrase(task, indicator)) {
|
|
395
398
|
return 'approximate';
|
|
396
399
|
}
|
|
397
400
|
}
|
|
@@ -405,20 +408,33 @@ export class CapabilityAnalyzer {
|
|
|
405
408
|
const relaxedIndicators = ['when possible', 'eventually', 'no rush'];
|
|
406
409
|
|
|
407
410
|
for (const indicator of strictIndicators) {
|
|
408
|
-
if (task
|
|
411
|
+
if (containsPhrase(task, indicator)) return 'strict';
|
|
409
412
|
}
|
|
410
413
|
for (const indicator of moderateIndicators) {
|
|
411
|
-
if (task
|
|
414
|
+
if (containsPhrase(task, indicator)) return 'moderate';
|
|
412
415
|
}
|
|
413
416
|
for (const indicator of relaxedIndicators) {
|
|
414
|
-
if (task
|
|
417
|
+
if (containsPhrase(task, indicator)) return 'relaxed';
|
|
415
418
|
}
|
|
416
419
|
|
|
417
420
|
return 'none';
|
|
418
421
|
}
|
|
419
422
|
|
|
420
|
-
private buildCacheKey(
|
|
421
|
-
|
|
423
|
+
private buildCacheKey(
|
|
424
|
+
task: string,
|
|
425
|
+
context?: {
|
|
426
|
+
availableTools?: Tool[];
|
|
427
|
+
previousTasks?: string[];
|
|
428
|
+
constraints?: { maxCost?: number; maxLatency?: number };
|
|
429
|
+
}
|
|
430
|
+
): string {
|
|
431
|
+
const raw = JSON.stringify([
|
|
432
|
+
task.toLowerCase().replace(/\s+/g, ' ').trim(),
|
|
433
|
+
(context?.availableTools ?? []).map((t) => t.name).sort(),
|
|
434
|
+
context?.previousTasks ?? [],
|
|
435
|
+
context?.constraints ?? {},
|
|
436
|
+
]);
|
|
437
|
+
return createHash('sha256').update(raw).digest('hex');
|
|
422
438
|
}
|
|
423
439
|
|
|
424
440
|
clearCache(): void {
|
|
@@ -10,6 +10,7 @@ export {
|
|
|
10
10
|
ParameterOptimizer,
|
|
11
11
|
type ParameterOptimizerOptions,
|
|
12
12
|
type OptimizationResult,
|
|
13
|
+
type HistoricalRecord,
|
|
13
14
|
} from './parameter-optimizer';
|
|
14
15
|
|
|
15
16
|
export {
|
|
@@ -19,5 +20,6 @@ export {
|
|
|
19
20
|
parseTaskProfileResponse,
|
|
20
21
|
parseCandidateGenerationResponse,
|
|
21
22
|
parsePerformanceAnalysisResponse,
|
|
23
|
+
sanitizeCandidateConfig,
|
|
22
24
|
ARCHITECTURE_ANALYSIS_SYSTEM_PROMPT,
|
|
23
25
|
} from './prompts';
|
|
@@ -13,17 +13,23 @@ import {
|
|
|
13
13
|
buildPerformanceAnalysisPrompt,
|
|
14
14
|
parseCandidateGenerationResponse,
|
|
15
15
|
parsePerformanceAnalysisResponse,
|
|
16
|
+
MAX_TOKENS_RANGE,
|
|
17
|
+
REFLECTION_DEPTH_RANGE,
|
|
18
|
+
TEMPERATURE_RANGE,
|
|
16
19
|
} from './prompts';
|
|
20
|
+
import { llmChat } from '../utils/llm-helper';
|
|
17
21
|
|
|
18
22
|
export interface ParameterOptimizerOptions {
|
|
19
23
|
llm: LLMBackend;
|
|
20
24
|
config: ArchitectureEvolutionConfig;
|
|
21
25
|
baseConfig: ArchitectureConfig;
|
|
22
26
|
model?: string;
|
|
27
|
+
availableModels?: string[];
|
|
23
28
|
}
|
|
24
29
|
|
|
25
30
|
export interface OptimizationResult {
|
|
26
31
|
recommendedConfig: ArchitectureConfig;
|
|
32
|
+
taskProfile: TaskProfile;
|
|
27
33
|
candidate: EvolutionCandidate | null;
|
|
28
34
|
metrics: EvolutionMetrics;
|
|
29
35
|
shouldAdopt: boolean;
|
|
@@ -31,7 +37,7 @@ export interface OptimizationResult {
|
|
|
31
37
|
reasoning: string;
|
|
32
38
|
}
|
|
33
39
|
|
|
34
|
-
interface HistoricalRecord {
|
|
40
|
+
export interface HistoricalRecord {
|
|
35
41
|
taskProfile: TaskProfile;
|
|
36
42
|
config: Partial<ArchitectureConfig>;
|
|
37
43
|
score: number;
|
|
@@ -44,6 +50,7 @@ export class ParameterOptimizer {
|
|
|
44
50
|
private readonly config: ArchitectureEvolutionConfig;
|
|
45
51
|
private readonly model: string;
|
|
46
52
|
private readonly baseConfig: ArchitectureConfig;
|
|
53
|
+
private readonly availableModels: string[];
|
|
47
54
|
private readonly capabilityAnalyzer: CapabilityAnalyzer;
|
|
48
55
|
private readonly evolutionStrategy: EvolutionStrategy;
|
|
49
56
|
|
|
@@ -58,6 +65,7 @@ export class ParameterOptimizer {
|
|
|
58
65
|
this.config = options.config;
|
|
59
66
|
this.model = options.model ?? 'default';
|
|
60
67
|
this.baseConfig = options.baseConfig;
|
|
68
|
+
this.availableModels = options.availableModels ?? [];
|
|
61
69
|
|
|
62
70
|
this.capabilityAnalyzer = new CapabilityAnalyzer({
|
|
63
71
|
llm: options.llm,
|
|
@@ -79,24 +87,28 @@ export class ParameterOptimizer {
|
|
|
79
87
|
}
|
|
80
88
|
|
|
81
89
|
const selection = this.evolutionStrategy.select(this.candidates);
|
|
90
|
+
const candidate = selection.candidate;
|
|
82
91
|
|
|
83
|
-
const mergedConfig = this.mergeConfigs(this.baseConfig,
|
|
92
|
+
const mergedConfig = this.mergeConfigs(this.baseConfig, candidate.config);
|
|
84
93
|
|
|
85
94
|
const bestEvaluated = this.candidates
|
|
86
95
|
.filter((c) => c.evaluationCount > 0)
|
|
87
96
|
.sort((a, b) => b.score - a.score)[0];
|
|
88
97
|
|
|
98
|
+
const threshold = this.config.adaptationThreshold ?? 0.1;
|
|
89
99
|
const shouldAdopt =
|
|
100
|
+
candidate.evaluationCount === 0 ||
|
|
90
101
|
!bestEvaluated ||
|
|
91
|
-
|
|
92
|
-
|
|
102
|
+
candidate.id === bestEvaluated.id ||
|
|
103
|
+
candidate.score >= bestEvaluated.score - threshold;
|
|
93
104
|
|
|
94
105
|
return {
|
|
95
106
|
recommendedConfig: mergedConfig,
|
|
96
|
-
|
|
107
|
+
taskProfile: profile,
|
|
108
|
+
candidate,
|
|
97
109
|
metrics: this.calculateMetrics(),
|
|
98
110
|
shouldAdopt,
|
|
99
|
-
confidence:
|
|
111
|
+
confidence: candidate.evaluationCount > 0 ? candidate.score : 0.5,
|
|
100
112
|
reasoning: selection.reasoning,
|
|
101
113
|
};
|
|
102
114
|
}
|
|
@@ -143,11 +155,13 @@ export class ParameterOptimizer {
|
|
|
143
155
|
const prompt = buildCandidateGenerationPrompt(
|
|
144
156
|
profile,
|
|
145
157
|
this.baseConfig,
|
|
146
|
-
this.getRelevantHistory(profile)
|
|
158
|
+
this.getRelevantHistory(profile),
|
|
159
|
+
{ availableModels: this.availableModels }
|
|
147
160
|
);
|
|
148
161
|
|
|
149
162
|
try {
|
|
150
|
-
const
|
|
163
|
+
const content = await llmChat(
|
|
164
|
+
this.llm,
|
|
151
165
|
[
|
|
152
166
|
{
|
|
153
167
|
role: 'system',
|
|
@@ -155,14 +169,17 @@ export class ParameterOptimizer {
|
|
|
155
169
|
},
|
|
156
170
|
{ role: 'user', content: prompt },
|
|
157
171
|
],
|
|
158
|
-
0.5
|
|
172
|
+
{ model: this.model, temperature: 0.5 }
|
|
159
173
|
);
|
|
160
174
|
|
|
161
|
-
const generated = parseCandidateGenerationResponse(
|
|
175
|
+
const generated = parseCandidateGenerationResponse(content, {
|
|
176
|
+
availableModels: this.availableModels,
|
|
177
|
+
}).filter((c) => c.risk !== 'high');
|
|
162
178
|
|
|
179
|
+
const maxCandidates = this.config.maxCandidates ?? 10;
|
|
163
180
|
this.candidates = [
|
|
164
181
|
this.createBaselineCandidate(),
|
|
165
|
-
...generated.map((c) => ({
|
|
182
|
+
...generated.slice(0, Math.max(0, maxCandidates - 1)).map((c) => ({
|
|
166
183
|
...c,
|
|
167
184
|
generation: this.currentGeneration,
|
|
168
185
|
})),
|
|
@@ -258,18 +275,27 @@ export class ParameterOptimizer {
|
|
|
258
275
|
|
|
259
276
|
if (Math.random() < mutationRate) {
|
|
260
277
|
const base = mutated.temperature ?? this.baseConfig.temperature;
|
|
261
|
-
mutated.temperature = Math.max(
|
|
278
|
+
mutated.temperature = Math.max(
|
|
279
|
+
TEMPERATURE_RANGE.min,
|
|
280
|
+
Math.min(TEMPERATURE_RANGE.max, base + (Math.random() - 0.5) * 0.3)
|
|
281
|
+
);
|
|
262
282
|
}
|
|
263
283
|
|
|
264
284
|
if (Math.random() < mutationRate) {
|
|
265
285
|
const base = mutated.maxTokens ?? this.baseConfig.maxTokens;
|
|
266
286
|
const delta = Math.floor((Math.random() - 0.5) * 1000);
|
|
267
|
-
mutated.maxTokens = Math.max(
|
|
287
|
+
mutated.maxTokens = Math.max(
|
|
288
|
+
MAX_TOKENS_RANGE.min,
|
|
289
|
+
Math.min(MAX_TOKENS_RANGE.max, base + delta)
|
|
290
|
+
);
|
|
268
291
|
}
|
|
269
292
|
|
|
270
293
|
if (Math.random() < mutationRate) {
|
|
271
294
|
const base = mutated.reflectionDepth ?? this.baseConfig.reflectionDepth;
|
|
272
|
-
mutated.reflectionDepth = Math.max(
|
|
295
|
+
mutated.reflectionDepth = Math.max(
|
|
296
|
+
REFLECTION_DEPTH_RANGE.min,
|
|
297
|
+
Math.min(REFLECTION_DEPTH_RANGE.max, base + Math.round(Math.random() * 2 - 1))
|
|
298
|
+
);
|
|
273
299
|
}
|
|
274
300
|
|
|
275
301
|
return mutated;
|
|
@@ -334,14 +360,14 @@ export class ParameterOptimizer {
|
|
|
334
360
|
if (minEvaluations < minRequired) return false;
|
|
335
361
|
|
|
336
362
|
const evaluationsSinceEvolution = this.history.filter(
|
|
337
|
-
(h) => h.timestamp > this.
|
|
363
|
+
(h) => h.timestamp > this.lastEvolutionTimestamp
|
|
338
364
|
).length;
|
|
339
365
|
|
|
340
|
-
return evaluationsSinceEvolution >=
|
|
366
|
+
return evaluationsSinceEvolution >= this.getEvaluationWindow();
|
|
341
367
|
}
|
|
342
368
|
|
|
343
|
-
private
|
|
344
|
-
return this.
|
|
369
|
+
private getEvaluationWindow(): number {
|
|
370
|
+
return Math.max(1, this.config.evaluationWindow ?? 10);
|
|
345
371
|
}
|
|
346
372
|
|
|
347
373
|
private getRelevantHistory(profile: TaskProfile): HistoricalRecord[] {
|
|
@@ -350,7 +376,7 @@ export class ParameterOptimizer {
|
|
|
350
376
|
(h) =>
|
|
351
377
|
h.taskProfile.domain === profile.domain || h.taskProfile.complexity === profile.complexity
|
|
352
378
|
)
|
|
353
|
-
.slice(-
|
|
379
|
+
.slice(-this.getEvaluationWindow());
|
|
354
380
|
}
|
|
355
381
|
|
|
356
382
|
private mergeConfigs(
|
|
@@ -446,9 +472,10 @@ export class ParameterOptimizer {
|
|
|
446
472
|
}
|
|
447
473
|
|
|
448
474
|
private calculateConvergenceRate(): number {
|
|
449
|
-
|
|
475
|
+
const window = this.getEvaluationWindow();
|
|
476
|
+
if (this.history.length < window) return 0;
|
|
450
477
|
|
|
451
|
-
const recentScores = this.history.slice(-
|
|
478
|
+
const recentScores = this.history.slice(-window).map((h) => h.score);
|
|
452
479
|
const variance = this.calculateVariance(recentScores);
|
|
453
480
|
|
|
454
481
|
return Math.max(0, 1 - variance * 4);
|
|
@@ -495,30 +522,21 @@ export class ParameterOptimizer {
|
|
|
495
522
|
const prompt = buildPerformanceAnalysisPrompt(evaluatedCandidates, results);
|
|
496
523
|
|
|
497
524
|
try {
|
|
498
|
-
const
|
|
525
|
+
const content = await llmChat(
|
|
526
|
+
this.llm,
|
|
499
527
|
[
|
|
500
528
|
{ role: 'system', content: 'You are an AI performance analyst.' },
|
|
501
529
|
{ role: 'user', content: prompt },
|
|
502
530
|
],
|
|
503
|
-
0.2
|
|
531
|
+
{ model: this.model, temperature: 0.2 }
|
|
504
532
|
);
|
|
505
533
|
|
|
506
|
-
return parsePerformanceAnalysisResponse(
|
|
534
|
+
return parsePerformanceAnalysisResponse(content);
|
|
507
535
|
} catch {
|
|
508
536
|
return null;
|
|
509
537
|
}
|
|
510
538
|
}
|
|
511
539
|
|
|
512
|
-
private async callLLM(
|
|
513
|
-
messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
|
|
514
|
-
temperature: number
|
|
515
|
-
) {
|
|
516
|
-
if (this.llm.complete) {
|
|
517
|
-
return this.llm.complete({ messages, temperature });
|
|
518
|
-
}
|
|
519
|
-
return this.llm.chat({ model: this.model, messages, temperature });
|
|
520
|
-
}
|
|
521
|
-
|
|
522
540
|
getCandidates(): EvolutionCandidate[] {
|
|
523
541
|
return [...this.candidates];
|
|
524
542
|
}
|
|
@@ -60,8 +60,18 @@ export function buildCandidateGenerationPrompt(
|
|
|
60
60
|
config: Partial<ArchitectureConfig>;
|
|
61
61
|
score: number;
|
|
62
62
|
metrics: Record<string, number>;
|
|
63
|
-
}
|
|
63
|
+
}>,
|
|
64
|
+
options: { availableModels?: string[] } = {}
|
|
64
65
|
): string {
|
|
66
|
+
const availableModels = options.availableModels ?? [];
|
|
67
|
+
const modelLine =
|
|
68
|
+
availableModels.length > 0
|
|
69
|
+
? `\n "model": ${availableModels.map((m) => JSON.stringify(m)).join(' | ')} | null,`
|
|
70
|
+
: '';
|
|
71
|
+
const modelRule =
|
|
72
|
+
availableModels.length > 0
|
|
73
|
+
? `Only use models from this list: ${availableModels.join(', ')}.`
|
|
74
|
+
: 'Do not change the model.';
|
|
65
75
|
const historySection = historicalPerformance?.length
|
|
66
76
|
? `
|
|
67
77
|
HISTORICAL PERFORMANCE:
|
|
@@ -85,18 +95,17 @@ ${historySection}
|
|
|
85
95
|
|
|
86
96
|
Generate 3-5 candidate configurations that might improve performance.
|
|
87
97
|
Each candidate should modify 1-3 parameters from current config.
|
|
98
|
+
${modelRule}
|
|
88
99
|
|
|
89
100
|
Respond with a JSON array:
|
|
90
101
|
[
|
|
91
102
|
{
|
|
92
103
|
"id": "candidate_1",
|
|
93
|
-
"config": {
|
|
94
|
-
"
|
|
95
|
-
"temperature": number or null,
|
|
104
|
+
"config": {${modelLine}
|
|
105
|
+
"temperature": number between 0 and 2 or null,
|
|
96
106
|
"maxTokens": number or null,
|
|
97
|
-
"systemPromptAdditions": "additional instructions or null",
|
|
98
107
|
"toolStrategy": "sequential" | "parallel" | "adaptive" | null,
|
|
99
|
-
"reflectionDepth":
|
|
108
|
+
"reflectionDepth": integer between 0 and 5 or null
|
|
100
109
|
},
|
|
101
110
|
"reasoning": "Why this configuration might help",
|
|
102
111
|
"expectedImprovement": 0.0-1.0,
|
|
@@ -142,58 +151,187 @@ Respond with:
|
|
|
142
151
|
}`;
|
|
143
152
|
}
|
|
144
153
|
|
|
154
|
+
const COMPLEXITIES: readonly TaskProfile['complexity'][] = [
|
|
155
|
+
'trivial',
|
|
156
|
+
'simple',
|
|
157
|
+
'moderate',
|
|
158
|
+
'complex',
|
|
159
|
+
'expert',
|
|
160
|
+
'extreme',
|
|
161
|
+
];
|
|
162
|
+
const DOMAINS: readonly TaskProfile['domain'][] = [
|
|
163
|
+
'general',
|
|
164
|
+
'coding',
|
|
165
|
+
'reasoning',
|
|
166
|
+
'creative',
|
|
167
|
+
'factual',
|
|
168
|
+
'conversational',
|
|
169
|
+
];
|
|
170
|
+
const TOOL_INTENSITIES: readonly TaskProfile['toolIntensity'][] = [
|
|
171
|
+
'none',
|
|
172
|
+
'light',
|
|
173
|
+
'moderate',
|
|
174
|
+
'heavy',
|
|
175
|
+
];
|
|
176
|
+
const REASONING_DEPTHS: readonly TaskProfile['reasoningDepth'][] = [
|
|
177
|
+
'shallow',
|
|
178
|
+
'moderate',
|
|
179
|
+
'deep',
|
|
180
|
+
'exhaustive',
|
|
181
|
+
];
|
|
182
|
+
const CREATIVITY_LEVELS: readonly TaskProfile['creativityLevel'][] = ['low', 'moderate', 'high'];
|
|
183
|
+
const ACCURACY_LEVELS: readonly TaskProfile['accuracyRequirement'][] = [
|
|
184
|
+
'approximate',
|
|
185
|
+
'moderate',
|
|
186
|
+
'high',
|
|
187
|
+
'critical',
|
|
188
|
+
];
|
|
189
|
+
const TIME_CONSTRAINTS: readonly TaskProfile['timeConstraint'][] = [
|
|
190
|
+
'none',
|
|
191
|
+
'relaxed',
|
|
192
|
+
'moderate',
|
|
193
|
+
'strict',
|
|
194
|
+
];
|
|
195
|
+
const TOOL_STRATEGIES: readonly ArchitectureConfig['toolStrategy'][] = [
|
|
196
|
+
'sequential',
|
|
197
|
+
'parallel',
|
|
198
|
+
'adaptive',
|
|
199
|
+
];
|
|
200
|
+
const RISKS: readonly EvolutionCandidate['risk'][] = ['low', 'medium', 'high'];
|
|
201
|
+
|
|
202
|
+
export const MAX_TOKENS_RANGE = { min: 100, max: 32000 } as const;
|
|
203
|
+
export const REFLECTION_DEPTH_RANGE = { min: 0, max: 5 } as const;
|
|
204
|
+
export const TEMPERATURE_RANGE = { min: 0, max: 2 } as const;
|
|
205
|
+
|
|
206
|
+
function pickEnum<T extends string>(value: unknown, allowed: readonly T[], fallback: T): T {
|
|
207
|
+
return typeof value === 'string' && (allowed as readonly string[]).includes(value)
|
|
208
|
+
? (value as T)
|
|
209
|
+
: fallback;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function finiteNumber(value: unknown): number | undefined {
|
|
213
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function clamp(value: number, range: { min: number; max: number }): number {
|
|
217
|
+
return Math.min(range.max, Math.max(range.min, value));
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
221
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
222
|
+
}
|
|
223
|
+
|
|
145
224
|
export function parseTaskProfileResponse(response: string): TaskProfile | null {
|
|
146
225
|
const json = extractJson(response);
|
|
147
226
|
if (!json) return null;
|
|
148
227
|
|
|
228
|
+
let parsed: unknown;
|
|
149
229
|
try {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
return {
|
|
153
|
-
complexity: parsed.complexity || 'moderate',
|
|
154
|
-
domain: parsed.domain || 'general',
|
|
155
|
-
estimatedTokens: parsed.estimatedTokens || 1000,
|
|
156
|
-
requiresTools: Boolean(parsed.requiresTools),
|
|
157
|
-
requiresReasoning: Boolean(parsed.requiresReasoning),
|
|
158
|
-
requiresCreativity: Boolean(parsed.requiresCreativity),
|
|
159
|
-
toolIntensity: parsed.toolIntensity || 'none',
|
|
160
|
-
reasoningDepth: parsed.reasoningDepth || 'moderate',
|
|
161
|
-
creativityLevel: parsed.creativityLevel || 'moderate',
|
|
162
|
-
accuracyRequirement: parsed.accuracyRequirement || 'moderate',
|
|
163
|
-
timeConstraint: parsed.timeConstraint || 'none',
|
|
164
|
-
};
|
|
230
|
+
parsed = JSON.parse(json);
|
|
165
231
|
} catch {
|
|
166
232
|
return null;
|
|
167
233
|
}
|
|
234
|
+
if (!isRecord(parsed)) return null;
|
|
235
|
+
|
|
236
|
+
const estimatedTokens = finiteNumber(parsed.estimatedTokens);
|
|
237
|
+
|
|
238
|
+
return {
|
|
239
|
+
complexity: pickEnum(parsed.complexity, COMPLEXITIES, 'moderate'),
|
|
240
|
+
domain: pickEnum(parsed.domain, DOMAINS, 'general'),
|
|
241
|
+
estimatedTokens:
|
|
242
|
+
estimatedTokens !== undefined && estimatedTokens > 0 ? Math.round(estimatedTokens) : 1000,
|
|
243
|
+
requiresTools: Boolean(parsed.requiresTools),
|
|
244
|
+
requiresReasoning: Boolean(parsed.requiresReasoning),
|
|
245
|
+
requiresCreativity: Boolean(parsed.requiresCreativity),
|
|
246
|
+
toolIntensity: pickEnum(parsed.toolIntensity, TOOL_INTENSITIES, 'none'),
|
|
247
|
+
reasoningDepth: pickEnum(parsed.reasoningDepth, REASONING_DEPTHS, 'moderate'),
|
|
248
|
+
creativityLevel: pickEnum(parsed.creativityLevel, CREATIVITY_LEVELS, 'moderate'),
|
|
249
|
+
accuracyRequirement: pickEnum(parsed.accuracyRequirement, ACCURACY_LEVELS, 'moderate'),
|
|
250
|
+
timeConstraint: pickEnum(parsed.timeConstraint, TIME_CONSTRAINTS, 'none'),
|
|
251
|
+
};
|
|
168
252
|
}
|
|
169
253
|
|
|
170
|
-
export function
|
|
254
|
+
export function sanitizeCandidateConfig(
|
|
255
|
+
raw: unknown,
|
|
256
|
+
options: { availableModels?: string[] } = {}
|
|
257
|
+
): Partial<ArchitectureConfig> {
|
|
258
|
+
if (!isRecord(raw)) return {};
|
|
259
|
+
const config: Partial<ArchitectureConfig> = {};
|
|
260
|
+
|
|
261
|
+
if (typeof raw.model === 'string' && options.availableModels?.includes(raw.model)) {
|
|
262
|
+
config.model = raw.model;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const temperature = finiteNumber(raw.temperature);
|
|
266
|
+
if (temperature !== undefined) {
|
|
267
|
+
config.temperature = clamp(temperature, TEMPERATURE_RANGE);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const maxTokens = finiteNumber(raw.maxTokens);
|
|
271
|
+
if (maxTokens !== undefined) {
|
|
272
|
+
config.maxTokens = clamp(Math.round(maxTokens), MAX_TOKENS_RANGE);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
if (
|
|
276
|
+
typeof raw.toolStrategy === 'string' &&
|
|
277
|
+
(TOOL_STRATEGIES as readonly string[]).includes(raw.toolStrategy)
|
|
278
|
+
) {
|
|
279
|
+
config.toolStrategy = raw.toolStrategy as ArchitectureConfig['toolStrategy'];
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
const reflectionDepth = finiteNumber(raw.reflectionDepth);
|
|
283
|
+
if (reflectionDepth !== undefined) {
|
|
284
|
+
config.reflectionDepth = clamp(Math.round(reflectionDepth), REFLECTION_DEPTH_RANGE);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
return config;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
export function parseCandidateGenerationResponse(
|
|
291
|
+
response: string,
|
|
292
|
+
options: { availableModels?: string[] } = {}
|
|
293
|
+
): EvolutionCandidate[] {
|
|
171
294
|
const json = extractJsonArray(response);
|
|
172
295
|
if (!json) return [];
|
|
173
296
|
|
|
297
|
+
let parsed: unknown;
|
|
174
298
|
try {
|
|
175
|
-
|
|
176
|
-
if (!Array.isArray(parsed)) return [];
|
|
177
|
-
|
|
178
|
-
return parsed
|
|
179
|
-
.filter((c: Record<string, unknown>) => c && typeof c === 'object' && c.config)
|
|
180
|
-
.map((c: Record<string, unknown>, idx: number) => ({
|
|
181
|
-
id: String(c.id || `candidate_${idx}`),
|
|
182
|
-
config: c.config as Partial<ArchitectureConfig>,
|
|
183
|
-
reasoning: String(c.reasoning || ''),
|
|
184
|
-
expectedImprovement:
|
|
185
|
-
typeof c.expectedImprovement === 'number' ? c.expectedImprovement : 0.5,
|
|
186
|
-
risk: (['low', 'medium', 'high'].includes(String(c.risk)) ? c.risk : 'medium') as
|
|
187
|
-
| 'low'
|
|
188
|
-
| 'medium'
|
|
189
|
-
| 'high',
|
|
190
|
-
generation: 0,
|
|
191
|
-
score: 0,
|
|
192
|
-
evaluationCount: 0,
|
|
193
|
-
}));
|
|
299
|
+
parsed = JSON.parse(json);
|
|
194
300
|
} catch {
|
|
195
301
|
return [];
|
|
196
302
|
}
|
|
303
|
+
if (!Array.isArray(parsed)) return [];
|
|
304
|
+
|
|
305
|
+
const usedIds = new Set<string>(['baseline']);
|
|
306
|
+
const candidates: EvolutionCandidate[] = [];
|
|
307
|
+
|
|
308
|
+
parsed.forEach((raw: unknown, idx: number) => {
|
|
309
|
+
if (!isRecord(raw)) return;
|
|
310
|
+
const config = sanitizeCandidateConfig(raw.config, options);
|
|
311
|
+
if (Object.keys(config).length === 0) return;
|
|
312
|
+
|
|
313
|
+
let id = typeof raw.id === 'string' && raw.id.trim() ? raw.id.trim() : `candidate_${idx}`;
|
|
314
|
+
while (usedIds.has(id)) {
|
|
315
|
+
id = `${id}_${idx}`;
|
|
316
|
+
}
|
|
317
|
+
usedIds.add(id);
|
|
318
|
+
|
|
319
|
+
const expectedImprovement = finiteNumber(raw.expectedImprovement);
|
|
320
|
+
|
|
321
|
+
candidates.push({
|
|
322
|
+
id,
|
|
323
|
+
config,
|
|
324
|
+
reasoning: typeof raw.reasoning === 'string' ? raw.reasoning : '',
|
|
325
|
+
expectedImprovement:
|
|
326
|
+
expectedImprovement !== undefined ? clamp(expectedImprovement, { min: 0, max: 1 }) : 0.5,
|
|
327
|
+
risk: pickEnum(raw.risk, RISKS, 'medium'),
|
|
328
|
+
generation: 0,
|
|
329
|
+
score: 0,
|
|
330
|
+
evaluationCount: 0,
|
|
331
|
+
});
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
return candidates;
|
|
197
335
|
}
|
|
198
336
|
|
|
199
337
|
export function parsePerformanceAnalysisResponse(response: string): {
|