@tyvm/knowhow 0.0.33 → 0.0.35

package/leaderboard/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "leaderboard",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev -p 3333",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "next": "^14.2.31",
+    "react": "^18",
+    "react-dom": "^18",
+    "recharts": "^2.8.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20",
+    "@types/react": "^18",
+    "@types/react-dom": "^18",
+    "autoprefixer": "^10.0.1",
+    "eslint": "^8",
+    "eslint-config-next": "14.0.0",
+    "postcss": "^8",
+    "tailwindcss": "^3.3.0",
+    "typescript": "^5"
+  }
+}

package/leaderboard/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

package/leaderboard/tailwind.config.js

@@ -0,0 +1,17 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    './pages/**/*.{js,ts,jsx,tsx,mdx}',
+    './components/**/*.{js,ts,jsx,tsx,mdx}',
+    './app/**/*.{js,ts,jsx,tsx,mdx}',
+  ],
+  theme: {
+    extend: {
+      colors: {
+        background: 'var(--background)',
+        foreground: 'var(--foreground)',
+      },
+    },
+  },
+  plugins: [],
+}

package/leaderboard/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "es6"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

package/leaderboard/types/benchmark.ts

@@ -0,0 +1,67 @@
+export interface BenchmarkConfig {
+  language: string;
+  maxExercises: number;
+  model: string;
+  provider: string;
+  agent?: string;
+  limits: BenchmarkLimits;
+  outputFile: string;
+}
+
+export interface BenchmarkLimits {
+  maxTurns: number;
+  maxTime: number;
+  maxCost: number;
+}
+
+export interface ExerciseResult {
+  exerciseName: string;
+  status: 'success' | 'failure' | 'timeout' | 'cost_limit' | 'turn_limit';
+  testResult?: any;
+  turns: number;
+  timeElapsed: number;
+  cost: number;
+  startTime: string;
+  endTime: string;
+  errorMessage?: string;
+  finalOutput?: string;
+}
+
+export interface BenchmarkResults {
+  config: BenchmarkConfig;
+  commitHash?: string;
+  exercises: ExerciseResult[];
+  summary: {
+    totalExercises: number;
+    testableExercises?: number;
+    testsPassedCount?: number;
+    testsFailedCount?: number;
+    testPassRate?: number;
+    agentSuccessRate?: number;
+    successCount: number;
+    failureCount: number;
+    timeoutCount: number;
+    costLimitCount: number;
+    turnLimitCount: number;
+    totalTime: number;
+    totalCost: number;
+    averageTurns: number;
+    averageTime: number;
+    successRate: number;
+  };
+  startTime: string;
+  endTime: string;
+}
+
+export interface LeaderboardEntry {
+  model: string;
+  provider: string;
+  language: string;
+  successRate: number;
+  totalExercises: number;
+  averageCost: number;
+  averageTime: number;
+  averageTurns: number;
+  totalRuns: number;
+  lastRun: string;
+}

package/leaderboard/utils/dataProcessor.ts

@@ -0,0 +1,33 @@
+import { LeaderboardEntry } from '@/types/benchmark';
+
+export async function loadLeaderboardData(): Promise<LeaderboardEntry[]> {
+  try {
+    const response = await fetch('/api/benchmark-data');
+    const data = await response.json();
+    return data;
+  } catch (error) {
+    console.error('Error loading benchmark results:', error);
+    return [];
+  }
+}
+
+export function formatPercentage(value: number): string {
+  return `${value.toFixed(1)}%`;
+}
+
+export function formatCurrency(value: number): string {
+  return `$${value.toFixed(3)}`;
+}
+
+export function formatTime(seconds: number): string {
+  if (seconds < 60) {
+    return `${seconds.toFixed(1)}s`;
+  }
+  const minutes = Math.floor(seconds / 60);
+  const remainingSeconds = seconds % 60;
+  return `${minutes}m ${remainingSeconds.toFixed(1)}s`;
+}
+
+export function formatNumber(value: number): string {
+  return value.toFixed(1);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.33",
+  "version": "0.0.35",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {
@@ -50,6 +50,7 @@
     "asana": "^3.0.16",
     "axios": "^1.5.0",
     "cheerio": "^1.0.0",
+    "commander": "^14.0.0",
     "diff": "^5.2.0",
     "esbuild": "^0.25.8",
     "express": "^4.19.2",

package/src/agents/base/base.ts

@@ -28,6 +28,7 @@ export interface AgentContext {
   Tools?: ToolsService;
   Events?: EventService;
   messageProcessor?: MessageProcessor;
+  Clients?: AIClient;
 }
 
 export abstract class BaseAgent implements IAgent {
@@ -43,6 +44,11 @@ export abstract class BaseAgent implements IAgent {
   protected currentModelPreferenceIndex = 0;
   protected easyFinalAnswer = false;
   protected requiredToolNames = ["finalAnswer"];
+  protected maxTurns: number | null = null;
+  protected maxSpend: number | null = null;
+  protected maxRunTimeMs: number | null = null;
+  protected startTimeMs: number | null = null;
+  protected turnCount = 0;
   protected totalCostUsd = 0;
   protected currentThread = 0;
   protected threads = [] as Message[][];
@@ -64,13 +70,15 @@ export abstract class BaseAgent implements IAgent {
   public tools: ToolsService;
   public events: EventService;
   public messageProcessor: MessageProcessor;
+  public clientService: AIClient;
 
   disabledTools = [];
 
   constructor(context: AgentContext) {
     this.tools = context.Tools;
     this.events = context.Events;
-    this.messageProcessor = context.messageProcessor;
+    this.messageProcessor = context.messageProcessor || new MessageProcessor();
+    this.clientService = context.Clients || Clients;
 
     if (!this.tools) {
       throw new Error("ToolsService is required for BaseAgent");
@@ -79,10 +87,18 @@ export abstract class BaseAgent implements IAgent {
     if (!this.events) {
       throw new Error("EventService is required for BaseAgent");
     }
+  }
 
-    if (!this.messageProcessor) {
-      this.messageProcessor = new MessageProcessor();
-    }
+  setMaxTurns(maxTurns: number | null) {
+    this.maxTurns = maxTurns;
+  }
+
+  setMaxSpend(maxSpend: number | null) {
+    this.maxSpend = maxSpend;
+  }
+
+  setMaxRunTime(maxRunTimeMs: number | null) {
+    this.maxRunTimeMs = maxRunTimeMs;
   }
 
   newTask() {
@@ -92,6 +108,8 @@ export abstract class BaseAgent implements IAgent {
     this.summaries = [];
     this.totalCostUsd = 0;
     this.status = "in_progress";
+    this.turnCount = 0;
+    this.startTimeMs = Date.now();
   }
 
   register() {
@@ -137,7 +155,8 @@ export abstract class BaseAgent implements IAgent {
 
   getClient() {
     if (!this.client) {
-      this.client = Clients.getClient(this.provider)?.client;
+      console.log("Getting client for provider", this.provider);
+      this.client = this.clientService.getClient(this.provider)?.client;
     }
     return this.client;
   }
@@ -174,6 +193,45 @@ export abstract class BaseAgent implements IAgent {
     }
   }
 
+  private checkLimits(): boolean {
+    // Check turn limit
+    if (this.maxTurns !== null && this.turnCount >= this.maxTurns) {
+      console.log(`Turn limit reached: ${this.turnCount}/${this.maxTurns}`);
+      return true;
+    }
+
+    // Check spend limit
+    if (this.maxSpend !== null && this.totalCostUsd >= this.maxSpend) {
+      console.log(
+        `Spend limit reached: $${this.totalCostUsd.toFixed(
+          4
+        )}/$${this.maxSpend.toFixed(4)}`
+      );
+      return true;
+    }
+
+    // Check runtime limit
+    if (this.maxRunTimeMs !== null && this.startTimeMs !== null) {
+      const currentRunTimeMs = Date.now() - this.startTimeMs;
+      if (currentRunTimeMs >= this.maxRunTimeMs) {
+        console.log(
+          `Runtime limit reached: ${currentRunTimeMs}ms/${this.maxRunTimeMs}ms`
+        );
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  private shouldTerminateFromLimits(): boolean {
+    return this.checkLimits();
+  }
+
+  getTurnCount(): number {
+    return this.turnCount;
+  }
+
   adjustTotalCostUsd(cost: number) {
     if (cost) {
       this.totalCostUsd += cost;
@@ -348,6 +406,27 @@ export abstract class BaseAgent implements IAgent {
     });
   }
 
+  async resume(resumeReason: string) {
+    const reason = resumeReason ? `Reason for resuming:  ${resumeReason}` : "";
+
+    // Create resume prompt
+    const resumePrompt = `We are resuming a previously started task. Here's the context:
+ORIGINAL REQUEST:
+${this.taskBreakdown}
+
+LAST Progress State:
+${JSON.stringify(this.threads[this.currentThread], null, 2)}
+
+Please continue from where you left off and complete the original request.
+${reason}
+
+`;
+
+    const lastThread = this.threads[this.currentThread] || [];
+    this.status = "in_progress";
+    this.call(resumePrompt, lastThread);
+  }
+
   async kill() {
     console.log("Killing agent");
     this.agentEvents.emit(this.eventTypes.kill, this);
@@ -366,6 +445,26 @@ export abstract class BaseAgent implements IAgent {
 
     await this.selectHealthyModel();
 
+    // Increment turn count and check limits (only for new calls, not recursive ones)
+    this.turnCount++;
+    if (this.shouldTerminateFromLimits()) {
+      const currentRunTimeMs = this.startTimeMs
+        ? Date.now() - this.startTimeMs
+        : 0;
+      const limitMsg = `Task terminated due to limits reached. Turn: ${
+        this.turnCount
+      }/${this.maxTurns || "unlimited"}, Cost: $${this.totalCostUsd.toFixed(
+        4
+      )}/${
+        this.maxSpend ? "$" + this.maxSpend.toFixed(4) : "unlimited"
+      }, Runtime: ${currentRunTimeMs}ms/${
+        this.maxRunTimeMs ? this.maxRunTimeMs + "ms" : "unlimited"
+      }`;
+      this.status = this.eventTypes.done;
+      this.agentEvents.emit(this.eventTypes.done, limitMsg);
+      return limitMsg;
+    }
+
     try {
       const model = this.getModel();
       let messages = _messages || (await this.getInitialMessages(userInput));
@@ -393,7 +492,7 @@ export abstract class BaseAgent implements IAgent {
       // Process messages before each AI call
       messages = await this.messageProcessor.processMessages(
         messages,
-        "per_call"
+        "pre_call"
       );
       const compressThreshold = 10000;
 
@@ -405,25 +504,36 @@ export abstract class BaseAgent implements IAgent {
       });
 
       if (response?.usd_cost === undefined) {
-        console.warn("Response cost is undefined", response);
+        console.warn(
+          "Response cost is undefined",
+          JSON.stringify(response, null, 2)
+        );
       }
 
       this.adjustTotalCostUsd(response?.usd_cost);
-      this.logMessages(response.choices.map((c) => c.message));
-
-      const firstMessage = response.choices[0].message;
-      const newToolCalls = response.choices.flatMap(
-        (c) => c.message.tool_calls
-      );
 
+      // Typically, there's only one choice in the array, but you could have many
+      // If you set `n` to more than 1, you will get multiple choices
       for (const choice of response.choices) {
-        const responseMessage = choice.message;
-        console.log(responseMessage);
+        messages.push(choice.message);
+
+        messages = await this.messageProcessor.processMessages(
+          messages,
+          "post_call"
+        );
+
+        const lastMessage = messages[messages.length - 1];
+
+        this.logMessages([lastMessage]);
 
-        const toolCalls = responseMessage.tool_calls;
-        if (responseMessage.tool_calls) {
-          // extend conversation with assistant's reply
-          messages.push(responseMessage);
+        const toolCalls = lastMessage.tool_calls;
+        if (lastMessage.tool_calls) {
+          // About to call a tool, process the messages
+          // We could add all the tool calls, and do this once
+          messages = await this.messageProcessor.processMessages(
+            messages,
+            "pre_tools"
+          );
 
           for (const toolCall of toolCalls) {
             const toolMessages = await this.processToolMessages(toolCall);
@@ -439,26 +549,32 @@ export abstract class BaseAgent implements IAgent {
             if (finalMessage) {
               const doneMsg = finalMessage.content || "Done";
               this.agentEvents.emit(this.eventTypes.done, doneMsg);
+              this.status = this.eventTypes.done;
               return doneMsg;
             }
           }
         }
       }
 
+      const newToolCalls = response.choices.flatMap(
+        (c) => c.message.tool_calls
+      );
       // Process messages after tool execution
       if (newToolCalls && newToolCalls.length > 0) {
         messages = await this.messageProcessor.processMessages(
           messages,
-          "post_call"
+          "post_tools"
         );
       }
 
       // Early exit: not required to call tool
+      const firstMessage = response.choices[0].message;
       if (
         response.choices.length === 1 &&
         firstMessage.content &&
         this.easyFinalAnswer
       ) {
+        this.status = this.eventTypes.done;
         this.agentEvents.emit(this.eventTypes.done, firstMessage.content);
         return firstMessage.content;
       }
@@ -469,14 +585,18 @@ export abstract class BaseAgent implements IAgent {
           this.requiredToolNames
         )} not available, options are ${this.getEnabledToolNames().join(", ")}`;
         console.error(error);
+        this.status = this.eventTypes.done;
         this.agentEvents.emit(this.eventTypes.done, error);
         return error;
       }
 
       // Early exit: killed, agent was requested to wrap up
-      if (this.pendingUserMessages.length === 0 && this.status === "killed") {
+      if (
+        this.pendingUserMessages.length === 0 &&
+        this.status === this.eventTypes.kill
+      ) {
         console.log("Agent killed, stopping execution");
-        this.status = "killed";
+        this.status = this.eventTypes.done;
         this.agentEvents.emit(this.eventTypes.done, firstMessage.content);
         return firstMessage.content;
       }
@@ -501,9 +621,43 @@ export abstract class BaseAgent implements IAgent {
         console.log(
           "Agent continuing to the next iteration, reminding agent how to terminate"
         );
+
+        const remainingTime =
+          this.maxRunTimeMs && this.startTimeMs
+            ? this.maxRunTimeMs - (Date.now() - this.startTimeMs)
+            : null;
+
+        const remainingTurns = this.maxTurns
+          ? this.maxTurns - this.turnCount
+          : null;
+
+        const timeRemainsingMsg = remainingTime
+          ? `You have approximately ${Math.floor(
+              remainingTime / 1000
+            )} seconds remaining for this task. `
+          : "";
+
+        const turnsRemainingMsg = remainingTurns
+          ? `You have ${remainingTurns} turns remaining. `
+          : "";
+
+        const remainingBudget = this.maxSpend
+          ? this.maxSpend - this.totalCostUsd
+          : null;
+        const budgetRemainingMsg = remainingBudget
+          ? `You have $${remainingBudget.toFixed(4)} remaining in your budget.`
+          : "";
+
+        const continuation = `<Workflow>
+        workflow continues until you call one of ${this.requiredToolNames}.\n
+        ${timeRemainsingMsg}
+        ${turnsRemainingMsg}
+        ${budgetRemainingMsg}
+        </Workflow>`;
+
         messages.push({
           role: "user",
-          content: `<Workflow>workflow continues until you call on of ${this.requiredToolNames}. Any continuation messages should be enclosed in workflow tags to hide intermediate work from users.</Workflow>`,
+          content: continuation,
         });
       }
 
@@ -522,7 +676,11 @@ export abstract class BaseAgent implements IAgent {
   }
 
   addPendingUserMessage(message: Message) {
-    this.pendingUserMessages.push(message);
+    if (this.status === this.eventTypes.done) {
+      this.resume(JSON.stringify(message.content));
+    } else {
+      this.pendingUserMessages.push(message);
+    }
   }
 
   getMessagesLength(messages: Message[]) {

package/src/agents/base/prompt.ts

@@ -17,6 +17,34 @@ You think step by step about the blocks of code you're modifying.
 You may use the execCommand tool to navigate the filesystem and to create new folders if needed.
 You may use the execCommand tool to use git to view which changes have been made so far via git diff and git status.
 
+# Creative Tool Usage Examples
+
+Beyond basic file editing, consider these powerful tool combinations:
+
+## String Replace Tool
+- **Bulk renaming**: Use stringReplace across multiple files to rename variables, functions, or imports
+- **Configuration updates**: Replace API endpoints, version numbers, or feature flags across the codebase
+- **Dependency updates**: Update import paths when reorganizing code structure
+
+## YCMD Language Server Integration
+- **Find all references**: Use ycmdGoTo with GoToReferences to find every usage of a function before refactoring
+- **Auto-complete exploration**: Use ycmdCompletion to discover available methods/properties on objects
+- **Smart navigation**: Use ycmdGoTo with GoToDefinition to understand code structure before making changes
+- **Error-driven development**: Use ycmdDiagnostics to identify compilation errors and fix them systematically
+- **Intelligent renaming**: Use ycmdRefactor with RefactorRename for safe symbol renaming across files
+
+## Search and Analysis Workflows
+- **Impact analysis**: Use textSearch + ycmdGoTo to understand how changes will affect the codebase
+- **Pattern discovery**: Use embeddingSearch to find similar code patterns when implementing new features
+- **Code exploration**: Combine fileSearch + readFile + ycmdCompletion to understand unfamiliar codebases
+
+## Advanced Combinations
+- **Refactoring workflow**: ycmdDiagnostics → ycmdGoTo → stringReplace → ycmdDiagnostics (verify fixes)
+- **Feature implementation**: embeddingSearch → ycmdCompletion → patchFile → ycmdDiagnostics
+- **Code cleanup**: textSearch → ycmdRefactor (organize imports) → stringReplace (standardize patterns)
+
+Think creatively about tool combinations - each tool can amplify the others' effectiveness!
+
 # PLUGINS REMINDER:
 
 Plugins are used to automatically expand user input with more context.

package/src/agents/index.ts

@@ -3,6 +3,7 @@ import { AgentContext } from "./base/base";
 import { DeveloperAgent } from "./developer/developer";
 import { PatchingAgent } from "./patcher/patcher";
 import { ResearcherAgent } from "./researcher/researcher";
+import { SetupAgent } from "./setup/setup";
 
 export { BaseAgent } from "./base/base";
 export { ConfigAgent } from "./configurable/ConfigAgent";
@@ -19,6 +20,7 @@ let singletons = {} as {
   Developer: DeveloperAgent;
   Patcher: PatchingAgent;
   Researcher: ResearcherAgent;
+  Setup: SetupAgent;
 };
 
 export function agents(agentContext: AgentContext = services()) {
@@ -27,6 +29,7 @@ export function agents(agentContext: AgentContext = services()) {
       Developer: new DeveloperAgent(agentContext),
       Patcher: new PatchingAgent(agentContext),
       Researcher: new ResearcherAgent(agentContext),
+      Setup: new SetupAgent(agentContext),
     };
   }
   return singletons;

package/src/agents/patcher/patcher.ts

@@ -77,13 +77,11 @@ export class PatchingAgent extends BaseAgent {
 
   constructor(context: AgentContext) {
     super(context);
-    this.disableTool("sendVimInput");
-    this.disableTool("openFileInVim");
 
     this.setModelPreferences([
       { model: Models.anthropic.Sonnet4, provider: "anthropic" },
       {
-        model: Models.openai.GPT_41,
+        model: Models.openai.GPT_41_Mini,
         provider: "openai",
       },
     ]);
@@ -98,10 +96,14 @@ export class PatchingAgent extends BaseAgent {
         ${systemReminder}
 
         Specialization: Patcher Agent, ${this.description}
+
+        # Language Server Integration:
+        Use ycmd tools (ycmdDiagnostics, ycmdCompletion, ycmdRefactor, ycmdGoTo) to get error diagnostics, code completions, refactor symbols, and navigate definitions before making changes.
+        This helps ensure accurate modifications and can suggest fixes for compilation errors.
+
         IF you fail twice to patch a file, you may switch using writeFileChunk to rewrite the whole file.
         `,
       },
-      // { role: "user", content: systemReminder },
       { role: "user", content: userInput },
     ] as Message[];
   }