@tyvm/knowhow 0.0.33 → 0.0.34

package/leaderboard/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "leaderboard",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev -p 3333",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "next": "^14.2.31",
+    "react": "^18",
+    "react-dom": "^18",
+    "recharts": "^2.8.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20",
+    "@types/react": "^18",
+    "@types/react-dom": "^18",
+    "autoprefixer": "^10.0.1",
+    "eslint": "^8",
+    "eslint-config-next": "14.0.0",
+    "postcss": "^8",
+    "tailwindcss": "^3.3.0",
+    "typescript": "^5"
+  }
+}

package/leaderboard/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

package/leaderboard/tailwind.config.js

@@ -0,0 +1,17 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    './pages/**/*.{js,ts,jsx,tsx,mdx}',
+    './components/**/*.{js,ts,jsx,tsx,mdx}',
+    './app/**/*.{js,ts,jsx,tsx,mdx}',
+  ],
+  theme: {
+    extend: {
+      colors: {
+        background: 'var(--background)',
+        foreground: 'var(--foreground)',
+      },
+    },
+  },
+  plugins: [],
+}

package/leaderboard/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "es6"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

package/leaderboard/types/benchmark.ts

@@ -0,0 +1,67 @@
+export interface BenchmarkConfig {
+  language: string;
+  maxExercises: number;
+  model: string;
+  provider: string;
+  agent?: string;
+  limits: BenchmarkLimits;
+  outputFile: string;
+}
+
+export interface BenchmarkLimits {
+  maxTurns: number;
+  maxTime: number;
+  maxCost: number;
+}
+
+export interface ExerciseResult {
+  exerciseName: string;
+  status: 'success' | 'failure' | 'timeout' | 'cost_limit' | 'turn_limit';
+  testResult?: any;
+  turns: number;
+  timeElapsed: number;
+  cost: number;
+  startTime: string;
+  endTime: string;
+  errorMessage?: string;
+  finalOutput?: string;
+}
+
+export interface BenchmarkResults {
+  config: BenchmarkConfig;
+  commitHash?: string;
+  exercises: ExerciseResult[];
+  summary: {
+    totalExercises: number;
+    testableExercises?: number;
+    testsPassedCount?: number;
+    testsFailedCount?: number;
+    testPassRate?: number;
+    agentSuccessRate?: number;
+    successCount: number;
+    failureCount: number;
+    timeoutCount: number;
+    costLimitCount: number;
+    turnLimitCount: number;
+    totalTime: number;
+    totalCost: number;
+    averageTurns: number;
+    averageTime: number;
+    successRate: number;
+  };
+  startTime: string;
+  endTime: string;
+}
+
+export interface LeaderboardEntry {
+  model: string;
+  provider: string;
+  language: string;
+  successRate: number;
+  totalExercises: number;
+  averageCost: number;
+  averageTime: number;
+  averageTurns: number;
+  totalRuns: number;
+  lastRun: string;
+}

package/leaderboard/utils/dataProcessor.ts

@@ -0,0 +1,33 @@
+import { LeaderboardEntry } from '@/types/benchmark';
+
+export async function loadLeaderboardData(): Promise<LeaderboardEntry[]> {
+  try {
+    const response = await fetch('/api/benchmark-data');
+    const data = await response.json();
+    return data;
+  } catch (error) {
+    console.error('Error loading benchmark results:', error);
+    return [];
+  }
+}
+
+export function formatPercentage(value: number): string {
+  return `${value.toFixed(1)}%`;
+}
+
+export function formatCurrency(value: number): string {
+  return `$${value.toFixed(3)}`;
+}
+
+export function formatTime(seconds: number): string {
+  if (seconds < 60) {
+    return `${seconds.toFixed(1)}s`;
+  }
+  const minutes = Math.floor(seconds / 60);
+  const remainingSeconds = seconds % 60;
+  return `${minutes}m ${remainingSeconds.toFixed(1)}s`;
+}
+
+export function formatNumber(value: number): string {
+  return value.toFixed(1);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.33",
+  "version": "0.0.34",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {
@@ -50,6 +50,7 @@
     "asana": "^3.0.16",
     "axios": "^1.5.0",
     "cheerio": "^1.0.0",
+    "commander": "^14.0.0",
     "diff": "^5.2.0",
     "esbuild": "^0.25.8",
     "express": "^4.19.2",

package/src/agents/base/base.ts

@@ -28,6 +28,7 @@ export interface AgentContext {
   Tools?: ToolsService;
   Events?: EventService;
   messageProcessor?: MessageProcessor;
+  Clients?: AIClient;
 }
 
 export abstract class BaseAgent implements IAgent {
@@ -43,6 +44,11 @@ export abstract class BaseAgent implements IAgent {
   protected currentModelPreferenceIndex = 0;
   protected easyFinalAnswer = false;
   protected requiredToolNames = ["finalAnswer"];
+  protected maxTurns: number | null = null;
+  protected maxSpend: number | null = null;
+  protected maxRunTimeMs: number | null = null;
+  protected startTimeMs: number | null = null;
+  protected turnCount = 0;
   protected totalCostUsd = 0;
   protected currentThread = 0;
   protected threads = [] as Message[][];
@@ -64,13 +70,15 @@ export abstract class BaseAgent implements IAgent {
   public tools: ToolsService;
   public events: EventService;
   public messageProcessor: MessageProcessor;
+  public clientService: AIClient;
 
   disabledTools = [];
 
   constructor(context: AgentContext) {
     this.tools = context.Tools;
     this.events = context.Events;
-    this.messageProcessor = context.messageProcessor;
+    this.messageProcessor = context.messageProcessor || new MessageProcessor();
+    this.clientService = context.Clients || Clients;
 
     if (!this.tools) {
       throw new Error("ToolsService is required for BaseAgent");
@@ -79,10 +87,18 @@ export abstract class BaseAgent implements IAgent {
     if (!this.events) {
       throw new Error("EventService is required for BaseAgent");
     }
+  }
 
-    if (!this.messageProcessor) {
-      this.messageProcessor = new MessageProcessor();
-    }
+  setMaxTurns(maxTurns: number | null) {
+    this.maxTurns = maxTurns;
+  }
+
+  setMaxSpend(maxSpend: number | null) {
+    this.maxSpend = maxSpend;
+  }
+
+  setMaxRunTime(maxRunTimeMs: number | null) {
+    this.maxRunTimeMs = maxRunTimeMs;
   }
 
   newTask() {
@@ -92,6 +108,8 @@ export abstract class BaseAgent implements IAgent {
     this.summaries = [];
     this.totalCostUsd = 0;
     this.status = "in_progress";
+    this.turnCount = 0;
+    this.startTimeMs = Date.now();
   }
 
   register() {
@@ -137,7 +155,8 @@ export abstract class BaseAgent implements IAgent {
 
   getClient() {
     if (!this.client) {
-      this.client = Clients.getClient(this.provider)?.client;
+      console.log("Getting client for provider", this.provider);
+      this.client = this.clientService.getClient(this.provider)?.client;
     }
     return this.client;
   }
@@ -174,6 +193,45 @@ export abstract class BaseAgent implements IAgent {
     }
   }
 
+  private checkLimits(): boolean {
+    // Check turn limit
+    if (this.maxTurns !== null && this.turnCount >= this.maxTurns) {
+      console.log(`Turn limit reached: ${this.turnCount}/${this.maxTurns}`);
+      return true;
+    }
+
+    // Check spend limit
+    if (this.maxSpend !== null && this.totalCostUsd >= this.maxSpend) {
+      console.log(
+        `Spend limit reached: $${this.totalCostUsd.toFixed(
+          4
+        )}/$${this.maxSpend.toFixed(4)}`
+      );
+      return true;
+    }
+
+    // Check runtime limit
+    if (this.maxRunTimeMs !== null && this.startTimeMs !== null) {
+      const currentRunTimeMs = Date.now() - this.startTimeMs;
+      if (currentRunTimeMs >= this.maxRunTimeMs) {
+        console.log(
+          `Runtime limit reached: ${currentRunTimeMs}ms/${this.maxRunTimeMs}ms`
+        );
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  private shouldTerminateFromLimits(): boolean {
+    return this.checkLimits();
+  }
+
+  getTurnCount(): number {
+    return this.turnCount;
+  }
+
   adjustTotalCostUsd(cost: number) {
     if (cost) {
       this.totalCostUsd += cost;
@@ -366,6 +424,25 @@ export abstract class BaseAgent implements IAgent {
 
     await this.selectHealthyModel();
 
+    // Increment turn count and check limits (only for new calls, not recursive ones)
+    this.turnCount++;
+    if (this.shouldTerminateFromLimits()) {
+      const currentRunTimeMs = this.startTimeMs
+        ? Date.now() - this.startTimeMs
+        : 0;
+      const limitMsg = `Task terminated due to limits reached. Turn: ${
+        this.turnCount
+      }/${this.maxTurns || "unlimited"}, Cost: $${this.totalCostUsd.toFixed(
+        4
+      )}/${
+        this.maxSpend ? "$" + this.maxSpend.toFixed(4) : "unlimited"
+      }, Runtime: ${currentRunTimeMs}ms/${
+        this.maxRunTimeMs ? this.maxRunTimeMs + "ms" : "unlimited"
+      }`;
+      this.agentEvents.emit(this.eventTypes.done, limitMsg);
+      return limitMsg;
+    }
+
     try {
       const model = this.getModel();
       let messages = _messages || (await this.getInitialMessages(userInput));
@@ -393,7 +470,7 @@ export abstract class BaseAgent implements IAgent {
       // Process messages before each AI call
       messages = await this.messageProcessor.processMessages(
         messages,
-        "per_call"
+        "pre_call"
       );
       const compressThreshold = 10000;
 
@@ -405,25 +482,36 @@ export abstract class BaseAgent implements IAgent {
       });
 
       if (response?.usd_cost === undefined) {
-        console.warn("Response cost is undefined", response);
+        console.warn(
+          "Response cost is undefined",
+          JSON.stringify(response, null, 2)
+        );
       }
 
       this.adjustTotalCostUsd(response?.usd_cost);
-      this.logMessages(response.choices.map((c) => c.message));
-
-      const firstMessage = response.choices[0].message;
-      const newToolCalls = response.choices.flatMap(
-        (c) => c.message.tool_calls
-      );
 
+      // Typically, there's only one choice in the array, but you could have many
+      // If you set `n` to more than 1, you will get multiple choices
       for (const choice of response.choices) {
-        const responseMessage = choice.message;
-        console.log(responseMessage);
+        messages.push(choice.message);
 
-        const toolCalls = responseMessage.tool_calls;
-        if (responseMessage.tool_calls) {
-          // extend conversation with assistant's reply
-          messages.push(responseMessage);
+        messages = await this.messageProcessor.processMessages(
+          messages,
+          "post_call"
+        );
+
+        const lastMessage = messages[messages.length - 1];
+
+        this.logMessages([lastMessage]);
+
+        const toolCalls = lastMessage.tool_calls;
+        if (lastMessage.tool_calls) {
+          // About to call a tool, process the messages
+          // We could add all the tool calls, and do this once
+          messages = await this.messageProcessor.processMessages(
+            messages,
+            "pre_tools"
+          );
 
           for (const toolCall of toolCalls) {
             const toolMessages = await this.processToolMessages(toolCall);
@@ -445,15 +533,19 @@ export abstract class BaseAgent implements IAgent {
         }
       }
 
+      const newToolCalls = response.choices.flatMap(
+        (c) => c.message.tool_calls
+      );
       // Process messages after tool execution
       if (newToolCalls && newToolCalls.length > 0) {
         messages = await this.messageProcessor.processMessages(
           messages,
-          "post_call"
+          "post_tools"
         );
       }
 
       // Early exit: not required to call tool
+      const firstMessage = response.choices[0].message;
       if (
         response.choices.length === 1 &&
         firstMessage.content &&
@@ -501,9 +593,43 @@ export abstract class BaseAgent implements IAgent {
         console.log(
           "Agent continuing to the next iteration, reminding agent how to terminate"
         );
+
+        const remainingTime =
+          this.maxRunTimeMs && this.startTimeMs
+            ? this.maxRunTimeMs - (Date.now() - this.startTimeMs)
+            : null;
+
+        const remainingTurns = this.maxTurns
+          ? this.maxTurns - this.turnCount
+          : null;
+
+        const timeRemainsingMsg = remainingTime
+          ? `You have approximately ${Math.floor(
+              remainingTime / 1000
+            )} seconds remaining for this task. `
+          : "";
+
+        const turnsRemainingMsg = remainingTurns
+          ? `You have ${remainingTurns} turns remaining. `
+          : "";
+
+        const remainingBudget = this.maxSpend
+          ? this.maxSpend - this.totalCostUsd
+          : null;
+        const budgetRemainingMsg = remainingBudget
+          ? `You have $${remainingBudget.toFixed(4)} remaining in your budget.`
+          : "";
+
+        const continuation = `<Workflow>
+        workflow continues until you call one of ${this.requiredToolNames}.\n
+        ${timeRemainsingMsg}
+        ${turnsRemainingMsg}
+        ${budgetRemainingMsg}
+        </Workflow>`;
+
         messages.push({
           role: "user",
-          content: `<Workflow>workflow continues until you call on of ${this.requiredToolNames}. Any continuation messages should be enclosed in workflow tags to hide intermediate work from users.</Workflow>`,
+          content: continuation,
         });
       }
 

package/src/agents/base/prompt.ts

@@ -17,6 +17,34 @@ You think step by step about the blocks of code you're modifying.
 You may use the execCommand tool to navigate the filesystem and to create new folders if needed.
 You may use the execCommand tool to use git to view which changes have been made so far via git diff and git status.
 
+# Creative Tool Usage Examples
+
+Beyond basic file editing, consider these powerful tool combinations:
+
+## String Replace Tool
+- **Bulk renaming**: Use stringReplace across multiple files to rename variables, functions, or imports
+- **Configuration updates**: Replace API endpoints, version numbers, or feature flags across the codebase
+- **Dependency updates**: Update import paths when reorganizing code structure
+
+## YCMD Language Server Integration
+- **Find all references**: Use ycmdGoTo with GoToReferences to find every usage of a function before refactoring
+- **Auto-complete exploration**: Use ycmdCompletion to discover available methods/properties on objects
+- **Smart navigation**: Use ycmdGoTo with GoToDefinition to understand code structure before making changes
+- **Error-driven development**: Use ycmdDiagnostics to identify compilation errors and fix them systematically
+- **Intelligent renaming**: Use ycmdRefactor with RefactorRename for safe symbol renaming across files
+
+## Search and Analysis Workflows
+- **Impact analysis**: Use textSearch + ycmdGoTo to understand how changes will affect the codebase
+- **Pattern discovery**: Use embeddingSearch to find similar code patterns when implementing new features
+- **Code exploration**: Combine fileSearch + readFile + ycmdCompletion to understand unfamiliar codebases
+
+## Advanced Combinations
+- **Refactoring workflow**: ycmdDiagnostics → ycmdGoTo → stringReplace → ycmdDiagnostics (verify fixes)
+- **Feature implementation**: embeddingSearch → ycmdCompletion → patchFile → ycmdDiagnostics
+- **Code cleanup**: textSearch → ycmdRefactor (organize imports) → stringReplace (standardize patterns)
+
+Think creatively about tool combinations - each tool can amplify the others' effectiveness!
+
 # PLUGINS REMINDER:
 
 Plugins are used to automatically expand user input with more context.

package/src/agents/index.ts

@@ -3,6 +3,7 @@ import { AgentContext } from "./base/base";
 import { DeveloperAgent } from "./developer/developer";
 import { PatchingAgent } from "./patcher/patcher";
 import { ResearcherAgent } from "./researcher/researcher";
+import { SetupAgent } from "./setup/setup";
 
 export { BaseAgent } from "./base/base";
 export { ConfigAgent } from "./configurable/ConfigAgent";
@@ -19,6 +20,7 @@ let singletons = {} as {
   Developer: DeveloperAgent;
   Patcher: PatchingAgent;
   Researcher: ResearcherAgent;
+  Setup: SetupAgent;
 };
 
 export function agents(agentContext: AgentContext = services()) {
@@ -27,6 +29,7 @@ export function agents(agentContext: AgentContext = services()) {
       Developer: new DeveloperAgent(agentContext),
       Patcher: new PatchingAgent(agentContext),
       Researcher: new ResearcherAgent(agentContext),
+      Setup: new SetupAgent(agentContext),
     };
   }
   return singletons;

package/src/agents/patcher/patcher.ts

@@ -77,13 +77,11 @@ export class PatchingAgent extends BaseAgent {
 
   constructor(context: AgentContext) {
     super(context);
-    this.disableTool("sendVimInput");
-    this.disableTool("openFileInVim");
 
     this.setModelPreferences([
       { model: Models.anthropic.Sonnet4, provider: "anthropic" },
       {
-        model: Models.openai.GPT_41,
+        model: Models.openai.GPT_41_Mini,
         provider: "openai",
       },
     ]);
@@ -98,10 +96,14 @@ export class PatchingAgent extends BaseAgent {
         ${systemReminder}
 
         Specialization: Patcher Agent, ${this.description}
+
+        # Language Server Integration:
+        Use ycmd tools (ycmdDiagnostics, ycmdCompletion, ycmdRefactor, ycmdGoTo) to get error diagnostics, code completions, refactor symbols, and navigate definitions before making changes.
+        This helps ensure accurate modifications and can suggest fixes for compilation errors.
+
         IF you fail twice to patch a file, you may switch using writeFileChunk to rewrite the whole file.
         `,
       },
-      // { role: "user", content: systemReminder },
       { role: "user", content: userInput },
     ] as Message[];
   }

package/src/agents/setup/setup.ts

@@ -0,0 +1,56 @@
+import { Message } from "../../clients/types";
+import { AgentContext, BaseAgent } from "../base/base";
+import { BASE_PROMPT } from "../base/prompt";
+import configExamples from "../../prompts/KnowhowConfigExamples";
+import { Models } from "../../ai";
+
+export class SetupAgent extends BaseAgent {
+  name = "Setup";
+  description = `This agent is great for setting up knowhow`;
+
+  constructor(context: AgentContext) {
+    super(context);
+
+    this.setModelPreferences([
+      { model: Models.anthropic.Sonnet4, provider: "anthropic" },
+      {
+        model: Models.openai.GPT_41_Mini,
+        provider: "openai",
+      },
+    ]);
+  }
+
+  async getInitialMessages(userInput: string) {
+    return [
+      {
+        role: "system",
+        content: `${BASE_PROMPT}
+
+        You are the Knowhow Setup agent, you work with the user to help setup their config to get the most out of knowhow.
+
+        Here are some config examples: ${configExamples}
+
+        Use the tools to analyze the codebase and help the user setup embeddings and mcp servers or any other config features, like linters etc
+
+        If the user asks about a mcp that we don't have in the config you can google it and figure out what the configuration would look like to add it.
+
+        If the task is for how to do something, you should ask the user if they want you to do it for them by using the askHuman tool. For isntance, how coul d I hook up this agent to gmail, you would research the mcp options for gmail, and then ask them if they'd like for you to update the config with whatever options you found.
+
+        Always ask the user to approve what you're going to do to the config, that way you can get feedback via askHuman before modifying the config
+
+        When setting up the language plugin for a user you should come up with phrases they're likely to say, like frontend/backend/schema etc that will signal we should load in guides or rules for that type of task. You should put any of your rules/analses in .knowhow/docs and the language plugin should reference those.
+
+        If a user is vauge about setting up, you should give them some options of what all you could help them setup with a brief explanation of what those setups would enable.
+
+        Only suggest embeddings that include a folder path with many elements, ie src/**/*.ts, never suggest entries with one element
+
+        If a user is requesting help with setting up a coding project, you can look at their package.json to setup the lintCommands so that we get feedback on file edits, and embeddings for the source code as those two features are the highest impact
+
+        If the user just says setup fast, try to get a general idea of the project file structure and setup one source code embedding for the whole codebaseand linter commands if possible. Try not do dig too deep if they want fast, just get the highest impact features setup
+
+        `,
+      },
+      { role: "user", content: userInput },
+    ] as Message[];
+  }
+}

package/src/agents/tools/agentCall.ts

@@ -7,8 +7,12 @@ export async function agentCall(agentName: string, userInput: string) {
     const toolService = this as ToolsService;
 
     const { Events, Plugins } = toolService.getContext();
-    const pluginText = await Plugins.callMany(config.plugins, userInput);
-    const fullPrompt = `${userInput} \n ${pluginText}`;
+
+    let fullPrompt = `${userInput}`;
+    if (config.plugins?.length) {
+      const pluginText = await Plugins.callMany(config.plugins, userInput);
+      fullPrompt += `\n ${pluginText}`;
+    }
 
     Events.emit("agents:call", {
       name: agentName,