@tyvm/knowhow 0.0.1

package/tests/integration/patching/input.txt

@@ -0,0 +1,145 @@
+import {
+  ChatCompletionMessageParam,
+  ChatCompletionToolMessageParam,
+  ChatCompletionMessageToolCall,
+} from "openai/resources/chat";
+import { openai } from "../../ai";
+import {
+  searchFiles,
+  scanFile,
+  writeFile,
+  applyPatchFile,
+  execCommand,
+  readFile,
+  finalAnswer,
+  addInternalTools,
+  callPlugin,
+  testPatch,
+} from "../tools";
+import { Tools } from "../tools/list";
+
+const availableFunctions = addInternalTools({
+  testPatch,
+  applyPatchFile,
+  callPlugin,
+  execCommand,
+  finalAnswer,
+  readFile,
+  scanFile,
+  searchFiles,
+  writeFile,
+});
+
+export class CodebaseAgent {
+  getInitialMessages(user_input: string) {
+    return [
+      {
+        role: "system",
+        content:
+          "Codebase Agent. You use the tools to read and write code, to help the developer implement features faster. Call final answer once you have finished implementing what is requested. As an agent you will receive multiple rounds of input until you call final answer. You are not able to request feedback from the user, so proceed with your plans and the developer will contact you afterwards if they need more help.",
+      },
+
+      { role: "user", content: user_input },
+    ] as Array<ChatCompletionMessageParam>;
+  }
+
+  async getToolMessages(toolCall: ChatCompletionMessageToolCall) {
+    const functionName = toolCall.function.name;
+    const functionToCall = availableFunctions[functionName];
+    const functionArgs = JSON.parse(toolCall.function.arguments);
+    const positionalArgs = Object.values(functionArgs);
+
+    console.log(
+      `Calling function ${functionName} with args:`,
+      JSON.stringify(positionalArgs, null, 2)
+    );
+
+    const functionResponse = await functionToCall(...positionalArgs);
+
+    if (functionName === "multi_tool_use.parallel") {
+      const args = positionalArgs[0] as {
+        recipient_name: string;
+        parameters: any;
+      }[];
+
+      return args.map((call, index) => {
+        return {
+          tool_call_id: toolCall.id + "_" + index,
+          role: "tool",
+          name: call.recipient_name.split(".").pop(),
+          content: functionResponse[index] || "Done",
+        };
+      });
+    }
+
+    const toolMessage = {
+      tool_call_id: toolCall.id,
+      role: "tool",
+      name: functionName,
+      content: functionResponse || "Done",
+    };
+
+    return [toolMessage];
+  }
+
+  logMessages(messages: Array<ChatCompletionMessageParam>) {
+    for (const message of messages) {
+      if (message.role == "assistant") {
+        console.log(message.content);
+      }
+    }
+  }
+
+  async call(
+    user_input: string,
+    _messages?: Array<ChatCompletionMessageParam>
+  ) {
+    const model = "gpt-4-1106-preview";
+    const messages = _messages || this.getInitialMessages(user_input);
+
+    const response = await openai.chat.completions.create({
+      model,
+      messages: messages,
+      tools: Tools,
+      tool_choice: "auto",
+    });
+    this.logMessages(response.choices.map((c) => c.message));
+    const responseMessage = response.choices[0].message;
+
+    const toolCalls = responseMessage.tool_calls;
+    if (responseMessage.tool_calls) {
+      // extend conversation with assistant's reply
+      messages.push(responseMessage);
+
+      for (const toolCall of toolCalls) {
+        const toolMessages = await this.getToolMessages(toolCall);
+        // Add the tool responses to the thread
+        messages.push(
+          ...(toolMessages as Array<ChatCompletionToolMessageParam>)
+        );
+        const finalMessage = toolMessages.find((m) => m.name === "finalAnswer");
+        if (finalMessage) {
+          return finalMessage.content;
+        }
+      }
+
+      // Send the tool responses back to the model
+      const secondResponse = await openai.chat.completions.create({
+        model,
+        messages: messages,
+      });
+
+      const aiResp = secondResponse.choices.map((c) => c.message);
+      this.logMessages(aiResp);
+      messages.push(...aiResp);
+
+      return this.call(user_input, messages);
+    }
+
+    if (responseMessage.content) {
+      return responseMessage.content;
+    }
+  }
+}
+
+export const Developer = new CodebaseAgent();

package/tests/integration/patching/output.txt

@@ -0,0 +1,145 @@
+import {
+  ChatCompletionMessageParam,
+  ChatCompletionToolMessageParam,
+  ChatCompletionMessageToolCall,
+} from "openai/resources/chat";
+import { openai } from "../../ai";
+import {
+  searchFiles,
+  scanFile,
+  writeFile,
+  applyPatchFile,
+  execCommand,
+  readFile,
+  finalAnswer,
+  addInternalTools,
+  callPlugin,
+  testPatch,
+} from "../tools";
+import { Tools } from "../tools/list";
+
+const availableFunctions = addInternalTools({
+  testPatch,
+  applyPatchFile,
+  callPlugin,
+  execCommand,
+  finalAnswer,
+  readFile,
+  scanFile,
+  searchFiles,
+  writeFile,
+});
+
+export class CodebaseAgent {
+  getInitialMessages(user_input: string) {
+    return [
+      {
+        role: "system",
+        content:
+          "Codebase Agent. You use the tools to read and write code, to help the developer implement features faster. Call final answer once you have finished implementing what is requested. As an agent you will receive multiple rounds of input until you call final answer. You are not able to request feedback from the user, so proceed with your plans and the developer will contact you afterwards if they need more help.",
+      },
+
+      { role: "user", content: user_input },
+    ] as Array<ChatCompletionMessageParam>;
+  }
+
+  async getToolMessages(toolCall: ChatCompletionMessageToolCall) {
+    const functionName = toolCall.function.name;
+    const functionToCall = availableFunctions[functionName];
+    const functionArgs = JSON.parse(toolCall.function.arguments);
+    const positionalArgs = Object.values(functionArgs);
+
+    console.log(
+      `Calling function ${functionName} with args:`,
+      JSON.stringify(positionalArgs, null, 2)
+    );
+
+    const functionResponse = await functionToCall(...positionalArgs);
+
+    if (functionName === "multi_tool_use.parallel") {
+      const args = positionalArgs[0] as {
+        recipient_name: string;
+        parameters: any;
+      }[];
+
+      return args.map((call, index) => {
+        return {
+          tool_call_id: toolCall.id + "_" + index,
+          role: "tool",
+          name: call.recipient_name.split(".").pop(),
+          content: functionResponse[index] || "Done",
+        };
+      });
+    }
+
+    const toolMessage = {
+      tool_call_id: toolCall.id,
+      role: "tool",
+      name: functionName,
+      content: functionResponse || "Done",
+    };
+
+    return [toolMessage];
+  }
+
+  logMessages(messages: Array<ChatCompletionMessageParam>) {
+    for (const message of messages) {
+      if (message.role == "assistant") {
+        console.log(message.content);
+      }
+    }
+  }
+
+  async call(
+    user_input: string,
+    _messages?: Array<ChatCompletionMessageParam>
+  ) {
+    const model = "gpt-4-1106-preview";
+    const messages = _messages || this.getInitialMessages(user_input);
+
+    const response = await openai.chat.completions.create({
+      model,
+      messages: messages,
+      tools: Tools,
+      tool_choice: "auto",
+    });
+    this.logMessages(response.choices.map((c) => c.message));
+    const responseMessage = response.choices[0].message;
+
+    const toolCalls = responseMessage.tool_calls;
+    if (responseMessage.tool_calls) {
+      // extend conversation with assistant's reply
+      messages.push(responseMessage);
+
+      for (const toolCall of toolCalls) {
+        const toolMessages = await this.getToolMessages(toolCall);
+        // Add the tool responses to the thread
+        messages.push(
+          ...(toolMessages as Array<ChatCompletionToolMessageParam>)
+        );
+        const finalMessage = toolMessages.find((m) => m.name === "FinalAnswer");
+        if (finalMessage) {
+          return finalMessage.content;
+        }
+      }
+
+      // Send the tool responses back to the model
+      const secondResponse = await openai.chat.completions.create({
+        model,
+        messages: messages,
+      });
+
+      const aiResp = secondResponse.choices.map((c) => c.message);
+      this.logMessages(aiResp);
+      messages.push(...aiResp);
+
+      return this.call(user_input, messages);
+    }
+
+    if (responseMessage.content) {
+      return responseMessage.content;
+    }
+  }
+}
+
+export const Developer = new CodebaseAgent();

package/tests/integration/patching/patch.txt

@@ -0,0 +1,15 @@
+Index: tests/integration/patching/input.txt
+===================================================================
+--- tests/integration/patching/input.txt
++++ tests/integration/patching/input.txt
+@@ -116,9 +116,9 @@
+         // Add the tool responses to the thread
+         messages.push(
+           ...(toolMessages as Array<ChatCompletionToolMessageParam>)
+         );
+-        const finalMessage = toolMessages.find((m) => m.name === "finalAnswer");
++        const finalMessage = toolMessages.find((m) => m.name === "FinalAnswer");
+         if (finalMessage) {
+           return finalMessage.content;
+         }
+       }

package/tests/integration/patching/unseen.txt

@@ -0,0 +1,80 @@
+import { Octokit } from "@octokit/rest";
+import { Plugin } from "./types";
+import parseDiff from "parse-diff";
+import { MinimalEmbedding } from "../types";
+
+export class GitHubPlugin implements Plugin {
+  octokit: Octokit;
+
+  constructor() {
+    this.octokit = new Octokit({
+      auth: process.env.GITHUB_TOKEN,
+    });
+  }
+
+  async embed(userPrompt: string): Promise<MinimalEmbedding[]> {
+    const urls = this.extractUrls(userPrompt);
+    const diffs = await this.getParsedDiffs(urls);
+    const diffsFiltered = diffs.filter((diff) => diff !== null);
+
+    return diffsFiltered.map((diff, index) => {
+      return {
+        id: urls[index],
+        text: JSON.stringify(diff),
+        metadata: {},
+      };
+    });
+  }
+
+  extractUrls(userPrompt: string): string[] {
+    const prUrlRegex =
+      /https:\/\/github\.com\/([\w-]+)\/([\w-]+)\/pull\/(\d+)/g;
+    const matches = userPrompt.match(prUrlRegex);
+    return matches;
+  }
+
+  async getDiff(url: string) {
+    const [owner, repo, _, pull_number] = url.split("/").slice(-4);
+    console.log(`Loading diff for ${owner}/${repo}#${pull_number}`);
+    const { data: diff } = await this.octokit.rest.pulls.get({
+      owner,
+      repo,
+      pull_number: parseInt(pull_number),
+      mediaType: {
+        format: "diff",
+      },
+    });
+
+    return diff;
+  }
+
+  async getParsedDiffs(urls: string[]) {
+    return Promise.all(
+      urls.map(async (url) => {
+        const diff = await this.getDiff(url);
+        let parsed = parseDiff(diff.toString());
+
+        parsed = parsed.filter((file) => {
+          return file.additions < 200 && file.deletions < 200;
+        });
+
+        return parsed;
+      })
+    );
+  }
+
+  async call(userPrompt: string): Promise<string> {
+    const urls = this.extractUrls(userPrompt);
+
+    if (urls) {
+      const responses = await this.getParsedDiffs(urls);
+      // Format the diffs in Markdown
+      const markdownDiffs = responses
+        .map((diff) => `\`\`\`diff\n${JSON.stringify(diff)}\n\`\`\``)
+        .join("\n\n");
+      return `GITHUB PLUGIN: ${urls} loaded:\n\n${markdownDiffs}`;
+    }
+
+    return "GITHUB PLUGIN: No pull request URLs detected.";
+  }
+}

package/tests/integration/patching.test.ts

@@ -0,0 +1,136 @@
+import { readFile, writeFile } from "../../src/utils";
+import { createPatch, applyPatch } from "diff";
+import { scanFile } from "../../src/agents/tools";
+import { patchFile } from "../../src/agents/tools/patch";
+import { Agents } from "../../src/services/AgentService";
+import { Patcher } from "../../src/agents";
+import { Tools } from "../../src/services";
+import { includedTools } from "../../src/agents/tools/list";
+import * as allTools from "../../src/agents/tools";
+
+const inputPath = "tests/integration/patching/input.txt";
+const outputPath = "tests/integration/patching/output.txt";
+const patchPath = "tests/integration/patching/patch.txt";
+
+const brokenPatch = `Index: tests/integration/patching/input.txt
+===================================================================
+--- tests/integration/patching/input.txt
++++ tests/integration/patching/input.txt
+@@ -116,9 +116,9 @@
+         // Add the tool responses to the thread
+         messages.push(
+           ...(toolMessages as Array<ChatCompletionToolMessageParam>)
+         );
+-        const finalMessage = toolMessages.find((m) => m.name === "finalAnswer");
++        const finalMessage = toolMessages.find((m) => m.name === "FinalAnswer");
+         if (finalMessage) {
+           return finalMessage.content;
+         }
+       }\n`;
+
+describe("Patcher", () => {
+  beforeAll(async () => {
+    Agents.registerAgent(Patcher);
+    Tools.addTools(includedTools);
+    const toolFunctions = Object.entries(allTools)
+      .filter(([_, value]) => typeof value === 'function')
+      .reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {});
+    Tools.addFunctions(toolFunctions);
+  });
+
+  test("should be able to apply actual patch", async () => {
+    const oldString = (await readFile(inputPath)).toString();
+    const newString = (await readFile(outputPath)).toString();
+    const workingPatch = (await readFile(patchPath)).toString();
+
+    const applied = await applyPatch(oldString, patchPath);
+
+    // Expect this not to work
+    expect(applied).toBe(oldString);
+
+    // Expect this to work
+    const workingApply = applyPatch(oldString, workingPatch);
+    expect(workingApply).toBe(newString);
+
+    console.log({
+      brokenPatch: Buffer.from(brokenPatch),
+      workingPatch: Buffer.from(workingPatch),
+    });
+
+    const buffer1 = Buffer.from(brokenPatch);
+    const buffer2 = Buffer.from(workingPatch);
+
+    console.log("bufferLength", buffer1.length, buffer2.length);
+    const length = Math.max(buffer1.length, buffer2.length);
+    for (let i = 0; i < length; i++) {
+      if (buffer1[i] !== buffer2[i]) {
+        console.log("not equal", i, buffer1[i], buffer2[i]);
+      }
+    }
+
+    expect(brokenPatch).toBe(workingPatch);
+  });
+
+  test("should generate the diff", async () => {
+    const oldString = (await readFile(inputPath)).toString();
+    const newString = (await readFile(outputPath)).toString();
+
+    const patch = createPatch(inputPath, oldString, newString);
+    await writeFile(patchPath, patch);
+
+    // const actualPatch = (await readFile(patchPath)).toString();
+
+    console.log(patch);
+
+    const scanned = await scanFile(inputPath, 120, 120);
+    console.log(JSON.parse(scanned));
+
+    expect(patch).toBe(brokenPatch);
+  });
+
+  test("should be able to patch a codebase", async () => {
+    const originalText = (await readFile(inputPath)).toString();
+    console.log(
+      await Agents.callAgent(
+        "Patcher",
+        `Update the file in ${inputPath} change the string "finalAnswer" to "FinalAnswer", as we are changing the tool name to be FinalAnswer instead of finalAnswer. This should be around line 120. Do not modify anything else. Treat this file as a typescript file. Only try one patch. If it fails stop`
+      )
+    );
+
+    const updatedText = (await readFile(inputPath)).toString();
+    await writeFile(inputPath, originalText);
+
+    expect(updatedText).toBe(
+      originalText.replace(`"finalAnswer"`, `"FinalAnswer"`)
+    );
+  });
+
+  it("should be able to patch when given the answer", async () => {
+    const originalText = (await readFile(inputPath)).toString();
+    await Agents.callAgent(
+      "Patcher",
+      `Update the file in ${inputPath} change the string "finalAnswer" to "FinalAnswer" on line 120. Do not modify anything else. Heres the answer: ${brokenPatch}.`
+    );
+
+    const updatedText = (await readFile(inputPath)).toString();
+    await writeFile(inputPath, originalText);
+
+    expect(updatedText).toBe(
+      originalText.replace(`"finalAnswer"`, `"FinalAnswer"`)
+    );
+  });
+
+  test("should be able to update the unseen file", async () => {
+    const unseenPath = "tests/integration/patching/unseen.txt";
+    const originalText = (await readFile(unseenPath)).toString();
+    await Agents.callAgent(
+      "Patcher",
+      `Update the file in ${unseenPath} change the file additions and deletions count from 200 to 300. Do not modify anything else. Treat this file as a typescript file.`
+    );
+
+    const updatedText = (await readFile(unseenPath)).toString();
+    await writeFile(unseenPath, originalText);
+
+    expect(updatedText).toBe(originalText.replaceAll("200", "300"));
+  });
+});

package/tests/languagePlugin.test.ts

@@ -0,0 +1,74 @@
+jest.mock("../src/utils", () => ({
+  readFile: jest.fn().mockReturnValue(Buffer.from("test")),
+  fileExists: jest.fn().mockReturnValue(true),
+  fileStat: jest.fn(),
+}));
+
+jest.mock("../src/config", () => ({
+  getConfig: jest.fn(),
+  getLanguageConfig: jest.fn(),
+}));
+
+jest.mock("../src/plugins/plugins", () => ({
+  Plugins: {
+    call: jest.fn().mockReturnValue("test"),
+    listPlugins: jest.fn().mockReturnValue(["github", "asana"]),
+  },
+}));
+
+import { LanguagePlugin } from "../src/plugins/language";
+import { Config } from "../src/types";
+import * as utils from "../src/utils";
+import { getConfig, getLanguageConfig } from "../src/config";
+import { Plugins } from "../src/plugins/plugins";
+
+const mockedConfig = getConfig as jest.MockedFunction<typeof getConfig>;
+const mockedLanguageConfig = getLanguageConfig as jest.MockedFunction<
+  typeof getLanguageConfig
+>;
+
+describe("LanguagePlugin", () => {
+  const userPrompt = "test prompt including terms";
+
+  test("should call the correct plugins based on the user prompt", async () => {
+    mockedConfig.mockResolvedValue({ plugins: ["github", "asana"] } as Config);
+    mockedLanguageConfig.mockResolvedValue({
+      test: {
+        sources: [
+          { kind: "github", data: ["http://github.com/test"] },
+          { kind: "asana", data: ["http://asana.com/test"] },
+          { kind: "file", data: ["../.knowhow/knowhow.json"] },
+        ],
+      },
+    });
+
+    const mockedPlugins = jest.mocked(Plugins);
+    const languagePlugin = new LanguagePlugin(mockedPlugins);
+    const pluginResponse = await languagePlugin.call(userPrompt);
+
+    expect(utils.fileExists).toHaveBeenCalled();
+    expect(utils.readFile).toHaveBeenCalled();
+    expect(mockedPlugins.listPlugins).toHaveBeenCalled();
+    expect(mockedPlugins.call).toHaveBeenCalledWith(
+      "github",
+      expect.any(String)
+    );
+    expect(mockedPlugins.call).toHaveBeenCalledWith(
+      "asana",
+      expect.any(String)
+    );
+    expect(pluginResponse).toContain(
+      "LANGUAGE PLUGIN: The following terms triggered expansions"
+    );
+  });
+
+  test("should return a message if no matching terms found", async () => {
+    mockedConfig.mockResolvedValue({ plugins: ["github"] } as Config);
+    mockedLanguageConfig.mockResolvedValue({});
+
+    const languagePlugin = new LanguagePlugin(Plugins);
+    const pluginResponse = await languagePlugin.call(userPrompt);
+
+    expect(pluginResponse).toEqual("LANGUAGE PLUGIN: No matching terms found");
+  });
+});

package/tests/patching/corrupted.test.ts

@@ -0,0 +1,78 @@
+import { fixPatch, hunksToPatch } from "../../src/agents/tools/patch";
+import { createTwoFilesPatch, applyPatch } from "diff";
+
+describe("fixPatch", () => {
+  it("should fix corrupted patches", async () => {
+    const original =
+      Array(10)
+        .fill(null)
+        .map(() => Math.random().toString(36).substring(2, 15))
+        .join("\n") + "\n";
+
+    // Create random change
+    const modified = original.split("\n");
+    modified[Math.floor(Math.random() * 10)] += "_changed";
+    const newString = modified.join("\n");
+
+    // Create patch
+    const patch = createTwoFilesPatch(
+      "file.txt",
+      "file.txt",
+      original,
+      newString
+    );
+
+    // Corrupt the header
+    let corruptedPatch = patch.replace(/@@.*/, "@@ -1000,1 +1000,1 @@");
+    const corruptedPatchLines = corruptedPatch.split("\n");
+
+    let incorrectLine = true;
+    let missingLine = true;
+    let sawSubtraction = false;
+    for (let i = 4; i < corruptedPatchLines.length; i++) {
+      const line = corruptedPatchLines[i];
+
+      if (line.startsWith("-")) {
+        sawSubtraction = true;
+        break;
+      }
+
+      if (line.startsWith(" ") && incorrectLine) {
+        corruptedPatchLines[i] = "XXXX";
+        incorrectLine = false;
+        continue;
+      }
+
+      if (line.startsWith(" ") && missingLine) {
+        delete corruptedPatchLines[i];
+        missingLine = false;
+        break;
+      }
+    }
+
+    corruptedPatch = corruptedPatchLines.join("\n");
+
+    // Try applying corrupted patch (should fail)
+    const attempt = applyPatch(original, corruptedPatch);
+    // expect(attempt).toBeUndefined();
+
+    // Fix the patch
+    console.log({ patch, corruptedPatch });
+    let fixedPatch = fixPatch(original, corruptedPatch);
+    console.log({ patch, corruptedPatch, attempt, fixedPatch });
+
+    // Apply fixed patch
+    const result = applyPatch(original, fixedPatch);
+
+    const header = fixedPatch.split("\n")[0];
+    const originalHeader = patch
+      .split("\n")
+      .find((line) => line.startsWith("@@"));
+
+    if (result !== newString) {
+      fixedPatch = fixPatch(original, corruptedPatch);
+    }
+
+    expect(result).toBe(newString);
+  });
+});

package/tests/patching/imports.patch.txt

@@ -0,0 +1,11 @@
+@@ -5,7 +6,7 @@
+ import React from 'react';
+ import { Col, Row } from 'react-bootstrap';
+
+-import { NameValueRow, NameValueRowWithBottomBorder } from '~/components/NameValueRow';
++import { NameValueRow } from '~/components/NameValueRow';
+ import { BOTTOM_BORDER_CONTAINER_STYLE } from '~/components/NameValueRow/NameValueRowWithBottomBorder';
+ import { NotesPopoverWrapper } from '~/components/Notes';
+ import { TokenAgreementRowAssetUnion, UpdateTokenAgreement, UpdateTokenAgreementVariables } from '~/generated/schema';
+
+