@cogitator-ai/self-modifying 0.1.0

package/src/tool-generation/gap-analyzer.ts

@@ -0,0 +1,234 @@
+import type {
+  Tool,
+  LLMBackend,
+  CapabilityGap,
+  GapAnalysisResult,
+  ToolSelfGenerationConfig,
+} from '@cogitator-ai/types';
+import { buildGapAnalysisPrompt, parseGapAnalysisResponse } from './prompts';
+
+export interface GapAnalyzerOptions {
+  llm: LLMBackend;
+  config: ToolSelfGenerationConfig;
+}
+
+export class GapAnalyzer {
+  private readonly llm: LLMBackend;
+  private readonly config: ToolSelfGenerationConfig;
+  private readonly analysisCache = new Map<string, GapAnalysisResult>();
+
+  constructor(options: GapAnalyzerOptions) {
+    this.llm = options.llm;
+    this.config = options.config;
+  }
+
+  async analyze(
+    userIntent: string,
+    availableTools: Tool[],
+    context?: {
+      failedAttempts?: string[];
+      previousGaps?: CapabilityGap[];
+    }
+  ): Promise<GapAnalysisResult> {
+    const cacheKey = this.buildCacheKey(userIntent, availableTools);
+    const cached = this.analysisCache.get(cacheKey);
+    if (cached && Date.now() - cached.timestamp.getTime() < 60000) {
+      return cached;
+    }
+
+    const toolSummaries = availableTools.map((t) => ({
+      name: t.name,
+      description: t.description,
+    }));
+
+    const prompt = buildGapAnalysisPrompt(
+      userIntent,
+      toolSummaries,
+      context?.failedAttempts
+    );
+
+    const response = await this.callLLM([
+      {
+        role: 'system',
+        content: `You are a capability analyzer for AI agents.
+Identify gaps between user intent and available tools.
+Be conservative - only report gaps when truly necessary.
+Consider tool composition before suggesting new tools.`,
+      },
+      { role: 'user', content: prompt },
+    ], 0.3);
+
+    const parsed = parseGapAnalysisResponse(response.content);
+    const filteredGaps = this.filterAndPrioritizeGaps(
+      parsed.gaps,
+      availableTools,
+      context?.previousGaps
+    );
+
+    const result: GapAnalysisResult = {
+      gaps: filteredGaps,
+      analysis: {
+        intentCoverage: this.calculateCoverage(userIntent, availableTools, filteredGaps),
+        suggestedCompositions: this.suggestCompositions(userIntent, availableTools),
+        canProceedWithExisting: parsed.canProceed && filteredGaps.length === 0,
+        reasoning: parsed.alternativeApproach || this.generateReasoning(filteredGaps),
+      },
+      timestamp: new Date(),
+    };
+
+    this.analysisCache.set(cacheKey, result);
+
+    if (this.analysisCache.size > 100) {
+      const oldest = Array.from(this.analysisCache.entries())
+        .sort((a, b) => a[1].timestamp.getTime() - b[1].timestamp.getTime())
+        .slice(0, 20);
+      oldest.forEach(([key]) => this.analysisCache.delete(key));
+    }
+
+    return result;
+  }
+
+  private filterAndPrioritizeGaps(
+    gaps: CapabilityGap[],
+    availableTools: Tool[],
+    previousGaps?: CapabilityGap[]
+  ): CapabilityGap[] {
+    const toolNames = new Set(availableTools.map((t) => t.name.toLowerCase()));
+    const previousGapIds = new Set(previousGaps?.map((g) => g.id) || []);
+
+    return gaps
+      .filter((gap) => {
+        if (toolNames.has(gap.suggestedToolName.toLowerCase())) {
+          return false;
+        }
+
+        if (gap.confidence < this.config.minConfidenceForGeneration) {
+          return false;
+        }
+
+        const complexityAllowed = this.isComplexityAllowed(gap.complexity);
+        if (!complexityAllowed) {
+          return false;
+        }
+
+        return true;
+      })
+      .map((gap) => ({
+        ...gap,
+        priority: this.calculatePriority(gap, previousGapIds.has(gap.id)),
+      }))
+      .sort((a, b) => (b.priority || 0) - (a.priority || 0))
+      .slice(0, this.config.maxToolsPerSession);
+  }
+
+  private isComplexityAllowed(complexity: 'simple' | 'moderate' | 'complex'): boolean {
+    const maxComplexity = this.config.maxComplexity || 'moderate';
+    const levels = { simple: 1, moderate: 2, complex: 3 };
+    return levels[complexity] <= levels[maxComplexity];
+  }
+
+  private calculatePriority(gap: CapabilityGap, isPreviouslyIdentified: boolean): number {
+    let priority = gap.confidence;
+
+    if (isPreviouslyIdentified) {
+      priority += 0.2;
+    }
+
+    const complexityBonus = { simple: 0.1, moderate: 0, complex: -0.1 };
+    priority += complexityBonus[gap.complexity];
+
+    return Math.min(1, Math.max(0, priority));
+  }
+
+  private calculateCoverage(
+    userIntent: string,
+    availableTools: Tool[],
+    gaps: CapabilityGap[]
+  ): number {
+    if (gaps.length === 0) {
+      return 1.0;
+    }
+
+    const intentWords = userIntent.toLowerCase().split(/\s+/).length;
+    const toolDescriptionWords = availableTools
+      .map((t) => t.description.toLowerCase().split(/\s+/))
+      .flat();
+
+    const intentTokens = new Set(userIntent.toLowerCase().split(/\s+/));
+    let matchedTokens = 0;
+
+    intentTokens.forEach((token) => {
+      if (token.length > 3 && toolDescriptionWords.includes(token)) {
+        matchedTokens++;
+      }
+    });
+
+    const baseCoverage = matchedTokens / Math.max(1, intentTokens.size);
+    const gapPenalty = gaps.reduce((sum, g) => sum + g.confidence * 0.2, 0);
+
+    return Math.max(0, Math.min(1, baseCoverage - gapPenalty));
+  }
+
+  private suggestCompositions(
+    userIntent: string,
+    availableTools: Tool[]
+  ): Array<{ tools: string[]; description: string }> {
+    const compositions: Array<{ tools: string[]; description: string }> = [];
+
+    const intentLower = userIntent.toLowerCase();
+
+    const keywords = {
+      transform: ['parse', 'format', 'convert'],
+      analyze: ['calculate', 'validate', 'check'],
+      combine: ['merge', 'join', 'aggregate'],
+    };
+
+    for (const [action, related] of Object.entries(keywords)) {
+      if (intentLower.includes(action)) {
+        const relatedTools = availableTools.filter((t) =>
+          related.some((r) => t.name.toLowerCase().includes(r) || t.description.toLowerCase().includes(r))
+        );
+
+        if (relatedTools.length >= 2) {
+          compositions.push({
+            tools: relatedTools.slice(0, 3).map((t) => t.name),
+            description: `Combine ${relatedTools.map((t) => t.name).join(' + ')} for ${action}`,
+          });
+        }
+      }
+    }
+
+    return compositions.slice(0, 3);
+  }
+
+  private generateReasoning(gaps: CapabilityGap[]): string {
+    if (gaps.length === 0) {
+      return 'All required capabilities are available with existing tools.';
+    }
+
+    const gapDescriptions = gaps
+      .map((g) => `- ${g.description} (confidence: ${(g.confidence * 100).toFixed(0)}%)`)
+      .join('\n');
+
+    return `Identified ${gaps.length} capability gap(s):\n${gapDescriptions}`;
+  }
+
+  private buildCacheKey(userIntent: string, tools: Tool[]): string {
+    const toolSignature = tools.map((t) => t.name).sort().join(',');
+    return `${userIntent.slice(0, 100)}|${toolSignature}`;
+  }
+
+  clearCache(): void {
+    this.analysisCache.clear();
+  }
+
+  private async callLLM(
+    messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
+    temperature: number
+  ) {
+    if (this.llm.complete) {
+      return this.llm.complete({ messages, temperature });
+    }
+    return this.llm.chat({ model: 'default', messages, temperature });
+  }
+}

package/src/tool-generation/generated-tool-store.ts

@@ -0,0 +1,268 @@
+import type { GeneratedTool, GeneratedToolStore as IGeneratedToolStore } from '@cogitator-ai/types';
+
+export interface ToolUsageRecord {
+  toolId: string;
+  timestamp: Date;
+  success: boolean;
+  executionTime: number;
+  inputSummary?: string;
+  error?: string;
+}
+
+export interface ToolMetrics {
+  totalUsage: number;
+  successCount: number;
+  failureCount: number;
+  averageExecutionTime: number;
+  lastUsed: Date | null;
+  successRate: number;
+}
+
+export class InMemoryGeneratedToolStore implements IGeneratedToolStore {
+  private readonly tools = new Map<string, GeneratedTool>();
+  private readonly usageRecords = new Map<string, ToolUsageRecord[]>();
+  private readonly maxUsageRecordsPerTool = 1000;
+
+  async save(tool: GeneratedTool): Promise<void> {
+    const existing = this.tools.get(tool.id);
+    if (existing) {
+      tool.version = existing.version + 1;
+    }
+    this.tools.set(tool.id, { ...tool, updatedAt: new Date() });
+  }
+
+  async get(id: string): Promise<GeneratedTool | null> {
+    return this.tools.get(id) || null;
+  }
+
+  async getByName(name: string): Promise<GeneratedTool | null> {
+    for (const tool of this.tools.values()) {
+      if (tool.name === name && tool.status !== 'deprecated') {
+        return tool;
+      }
+    }
+    return null;
+  }
+
+  async list(filter?: {
+    status?: GeneratedTool['status'];
+    minScore?: number;
+  }): Promise<GeneratedTool[]> {
+    let tools = Array.from(this.tools.values());
+
+    if (filter?.status) {
+      tools = tools.filter((t) => t.status === filter.status);
+    }
+
+    if (filter?.minScore !== undefined) {
+      tools = tools.filter(
+        (t) => t.validationScore !== undefined && t.validationScore >= filter.minScore!
+      );
+    }
+
+    return tools.sort((a, b) => {
+      const scoreA = a.validationScore ?? 0;
+      const scoreB = b.validationScore ?? 0;
+      return scoreB - scoreA;
+    });
+  }
+
+  async delete(id: string): Promise<boolean> {
+    const deleted = this.tools.delete(id);
+    this.usageRecords.delete(id);
+    return deleted;
+  }
+
+  async updateStatus(
+    id: string,
+    status: GeneratedTool['status']
+  ): Promise<boolean> {
+    const tool = this.tools.get(id);
+    if (!tool) return false;
+
+    tool.status = status;
+    tool.updatedAt = new Date();
+    return true;
+  }
+
+  async recordUsage(record: ToolUsageRecord): Promise<void> {
+    const records = this.usageRecords.get(record.toolId) || [];
+    records.push(record);
+
+    if (records.length > this.maxUsageRecordsPerTool) {
+      records.splice(0, records.length - this.maxUsageRecordsPerTool);
+    }
+
+    this.usageRecords.set(record.toolId, records);
+
+    const tool = this.tools.get(record.toolId);
+    if (tool) {
+      tool.usageCount = (tool.usageCount || 0) + 1;
+      tool.lastUsed = record.timestamp;
+    }
+  }
+
+  async getMetrics(toolId: string): Promise<ToolMetrics | null> {
+    const records = this.usageRecords.get(toolId);
+    if (!records || records.length === 0) {
+      return null;
+    }
+
+    const successCount = records.filter((r) => r.success).length;
+    const totalTime = records.reduce((sum, r) => sum + r.executionTime, 0);
+
+    return {
+      totalUsage: records.length,
+      successCount,
+      failureCount: records.length - successCount,
+      averageExecutionTime: totalTime / records.length,
+      lastUsed: records.length > 0 ? records[records.length - 1].timestamp : null,
+      successRate: successCount / records.length,
+    };
+  }
+
+  async getTopTools(limit: number = 10): Promise<Array<{ tool: GeneratedTool; metrics: ToolMetrics }>> {
+    const result: Array<{ tool: GeneratedTool; metrics: ToolMetrics }> = [];
+
+    for (const tool of this.tools.values()) {
+      if (tool.status !== 'active') continue;
+
+      const metrics = await this.getMetrics(tool.id);
+      if (metrics && metrics.totalUsage > 0) {
+        result.push({ tool, metrics });
+      }
+    }
+
+    return result
+      .sort((a, b) => {
+        const scoreA = a.metrics.successRate * Math.log(a.metrics.totalUsage + 1);
+        const scoreB = b.metrics.successRate * Math.log(b.metrics.totalUsage + 1);
+        return scoreB - scoreA;
+      })
+      .slice(0, limit);
+  }
+
+  async findSimilar(description: string, limit: number = 5): Promise<GeneratedTool[]> {
+    const descWords = new Set(description.toLowerCase().split(/\s+/));
+    const scored: Array<{ tool: GeneratedTool; score: number }> = [];
+
+    for (const tool of this.tools.values()) {
+      if (tool.status === 'deprecated') continue;
+
+      const toolWords = new Set(
+        `${tool.name} ${tool.description}`.toLowerCase().split(/\s+/)
+      );
+
+      let matchCount = 0;
+      for (const word of descWords) {
+        if (word.length >= 3 && toolWords.has(word)) {
+          matchCount++;
+        }
+      }
+
+      if (matchCount > 0) {
+        scored.push({
+          tool,
+          score: matchCount / Math.max(descWords.size, toolWords.size),
+        });
+      }
+    }
+
+    return scored
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit)
+      .map((s) => s.tool);
+  }
+
+  async promoteToActive(toolId: string): Promise<boolean> {
+    const tool = this.tools.get(toolId);
+    if (!tool) return false;
+
+    if (tool.status !== 'validated') {
+      return false;
+    }
+
+    tool.status = 'active';
+    tool.updatedAt = new Date();
+    return true;
+  }
+
+  async deprecate(toolId: string, reason?: string): Promise<boolean> {
+    const tool = this.tools.get(toolId);
+    if (!tool) return false;
+
+    tool.status = 'deprecated';
+    tool.metadata = {
+      ...tool.metadata,
+      deprecationReason: reason,
+      deprecatedAt: new Date().toISOString(),
+    };
+    tool.updatedAt = new Date();
+    return true;
+  }
+
+  async cleanup(options: {
+    maxAge?: number;
+    minUsage?: number;
+    maxDeprecated?: number;
+  }): Promise<number> {
+    let removed = 0;
+    const now = Date.now();
+
+    for (const [id, tool] of this.tools) {
+      if (options.maxAge && tool.createdAt) {
+        const age = now - tool.createdAt.getTime();
+        if (age > options.maxAge && tool.status !== 'active') {
+          this.tools.delete(id);
+          this.usageRecords.delete(id);
+          removed++;
+          continue;
+        }
+      }
+
+      if (options.minUsage !== undefined) {
+        const usage = tool.usageCount || 0;
+        if (usage < options.minUsage && tool.status === 'deprecated') {
+          this.tools.delete(id);
+          this.usageRecords.delete(id);
+          removed++;
+        }
+      }
+    }
+
+    if (options.maxDeprecated !== undefined) {
+      const deprecated = Array.from(this.tools.values())
+        .filter((t) => t.status === 'deprecated')
+        .sort((a, b) => (a.usageCount || 0) - (b.usageCount || 0));
+
+      while (deprecated.length > options.maxDeprecated) {
+        const toRemove = deprecated.shift()!;
+        this.tools.delete(toRemove.id);
+        this.usageRecords.delete(toRemove.id);
+        removed++;
+      }
+    }
+
+    return removed;
+  }
+
+  getStats(): {
+    total: number;
+    byStatus: Record<string, number>;
+    totalUsage: number;
+  } {
+    const byStatus: Record<string, number> = {};
+    let totalUsage = 0;
+
+    for (const tool of this.tools.values()) {
+      byStatus[tool.status] = (byStatus[tool.status] || 0) + 1;
+      totalUsage += tool.usageCount || 0;
+    }
+
+    return {
+      total: this.tools.size,
+      byStatus,
+      totalUsage,
+    };
+  }
+}

package/src/tool-generation/index.ts

@@ -0,0 +1,19 @@
+export { GapAnalyzer, type GapAnalyzerOptions } from './gap-analyzer';
+export { ToolGenerator, type ToolGeneratorOptions, type GenerationResult } from './tool-generator';
+export { ToolValidator, type ToolValidatorOptions } from './tool-validator';
+export { ToolSandbox, DEFAULT_SANDBOX_CONFIG } from './tool-sandbox';
+export {
+  InMemoryGeneratedToolStore,
+  type ToolUsageRecord,
+  type ToolMetrics,
+} from './generated-tool-store';
+export {
+  buildGapAnalysisPrompt,
+  buildToolGenerationPrompt,
+  buildToolValidationPrompt,
+  buildToolImprovementPrompt,
+  parseGapAnalysisResponse,
+  parseToolGenerationResponse,
+  parseValidationResponse,
+  TOOL_GENERATION_SYSTEM_PROMPT,
+} from './prompts';