@tyvm/knowhow 0.0.1

package/src/config.ts

@@ -0,0 +1,211 @@
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+import gitignoreToGlob from "gitignore-to-glob";
+import { Prompts } from "./prompts";
+import { promisify } from "util";
+import { Config, Language, AssistantConfig, Models } from "./types";
+import { mkdir, writeFile, readFile, fileExists } from "./utils";
+
+const defaultConfig = {
+  promptsDir: ".knowhow/prompts",
+  modules: [],
+  plugins: [
+    "embeddings",
+    "language",
+    "vim",
+    "github",
+    "asana",
+    "jira",
+    "linear",
+    "download",
+    "figma",
+    "url",
+  ],
+  lintCommands: {
+    js: "eslint",
+    ts: "tslint",
+  },
+  sources: [
+    {
+      input: "src/**/*.mdx",
+      output: ".knowhow/docs/",
+      prompt: "BasicCodeDocumenter",
+    },
+    {
+      input: ".knowhow/docs/**/*.mdx",
+      output: ".knowhow/docs/README.mdx",
+      prompt: "BasicProjectDocumenter",
+    },
+  ],
+  embedSources: [
+    {
+      input: ".knowhow/docs/**/*.mdx",
+      output: ".knowhow/embeddings/docs.json",
+      prompt: "BasicEmbeddingExplainer",
+      chunkSize: 2000,
+    },
+  ],
+  embeddingModel: Models.openai.EmbeddingAda2,
+
+  agents: [
+    {
+      name: "Example agent",
+      description:
+        "You can define agents in the config. They will have access to all tools.",
+      instructions: "Reply to the user saying 'Hello, world!'",
+      model: "gpt-4o-2024-08-06",
+      provider: "openai",
+    },
+  ],
+  mcps: [
+    {
+      name: "browser",
+      command: "npx",
+      args: ["-y", "@playwright/mcp@latest", "--browser", "chrome"],
+    },
+  ],
+
+  modelProviders: [{ url: "http://localhost:1234", provider: "lms" }],
+} as Config;
+
+const defaultLanguage = {
+  "knowhow config": {
+    sources: [
+      {
+        kind: "file",
+        data: [".knowhow/knowhow.json"],
+      },
+    ],
+  },
+} as Language;
+
+const globalTemplateFolders = ["prompts", "docs", "embeddings"];
+const globalTemplateFiles = {
+  "knowhow.json": JSON.stringify(defaultConfig, null, 2),
+  "language.json": JSON.stringify(defaultLanguage, null, 2),
+  ".ignore": "",
+  ".hashes.json": "{}",
+  ".jwt": "",
+};
+
+for (const prompt of Prompts) {
+  const promptName = Object.keys(prompt)[0];
+  const fileName = "prompts/" + promptName + ".mdx";
+  globalTemplateFiles[fileName] = prompt[promptName];
+}
+
+function getGlobalConfigDir() {
+  return path.join(os.homedir(), ".knowhow");
+}
+
+async function ensureGlobalConfigDir() {
+  const globalConfigDir = getGlobalConfigDir();
+  await mkdir(globalConfigDir, { recursive: true });
+
+  for (const folder of globalTemplateFolders) {
+    const folderPath = path.join(globalConfigDir, folder);
+    await mkdir(folderPath, { recursive: true });
+    fs.chmodSync(folderPath, 0o600);
+  }
+
+  for (const file of Object.keys(globalTemplateFiles)) {
+    const filePath = path.join(globalConfigDir, file);
+    if (!fs.existsSync(filePath)) {
+      await writeFile(filePath, globalTemplateFiles[file]);
+      fs.chmodSync(filePath, 0o600);
+    }
+  }
+  return globalConfigDir;
+}
+
+export async function init() {
+  const globalConfigDir = await ensureGlobalConfigDir();
+
+  // create the folder structure
+  await mkdir(".knowhow", { recursive: true });
+  for (const folder of globalTemplateFolders) {
+    await mkdir(path.join(".knowhow", folder), { recursive: true });
+  }
+
+  // Copy the template prompts
+  await copyTemplates(globalConfigDir);
+}
+
+export async function getLanguageConfig() {
+  const language = JSON.parse(await readFile(".knowhow/language.json", "utf8"));
+  return language as Language;
+}
+
+export async function updateLanguageConfig(language: Language) {
+  await writeFile(".knowhow/language.json", JSON.stringify(language, null, 2));
+}
+
+export async function updateConfig(config: Config) {
+  await writeFile(".knowhow/knowhow.json", JSON.stringify(config, null, 2));
+}
+
+async function copyTemplates(globalConfigDir: string) {
+  for (const folder of globalTemplateFolders) {
+    const src = path.join(globalConfigDir, folder);
+    const dest = path.join(".knowhow", folder);
+    // Copy all the template folders, skipping files that already exist
+    await fs.promises.cp(src, dest, {
+      recursive: true,
+      filter: (source, destination) => {
+        return fs.existsSync(dest) ? false : true;
+      },
+    });
+  }
+
+  // Copy the template files, skipping files that already exist
+  for (const file of Object.keys(globalTemplateFiles)) {
+    const src = path.join(globalConfigDir, file);
+    const dest = path.join(".knowhow", file);
+    if (!fs.existsSync(dest)) {
+      await fs.promises.copyFile(src, dest);
+    }
+  }
+}
+
+export function getConfigSync() {
+  try {
+    const config = JSON.parse(
+      fs.readFileSync(".knowhow/knowhow.json", "utf8").toString()
+    );
+    return config as Config;
+  } catch (e) {
+    return {} as Config;
+  }
+}
+
+export async function getConfig() {
+  const config = JSON.parse(await readFile(".knowhow/knowhow.json", "utf8"));
+  return config as Config;
+}
+
+export async function loadPrompt(promptName: string) {
+  const config = await getConfig();
+  if (!promptName) {
+    return "";
+  }
+
+  const prompt = await readFile(
+    path.join(config.promptsDir, `${promptName}.mdx`),
+    "utf8"
+  );
+  return prompt;
+}
+
+export async function getIgnorePattern() {
+  const ignoreList = new Array<string>();
+  const gitIgnore = await fileExists(".gitignore");
+  if (gitIgnore) {
+    ignoreList.push(...gitignoreToGlob(".gitignore"));
+  }
+  const knowhowIgnore = await fileExists(".knowhow/.ignore");
+  if (knowhowIgnore) {
+    ignoreList.push(...gitignoreToGlob(".knowhow/.ignore"));
+  }
+  return ignoreList.map((pattern) => pattern.replace("!", "./"));
+}

package/src/conversion.ts

@@ -0,0 +1,140 @@
+import pdf from "pdf-parse";
+import * as fs from "fs";
+import * as path from "path";
+import { readFile, fileExists } from "./utils";
+import { Downloader } from "./plugins/downloader/downloader";
+
+export async function processAudio(
+  filePath: string,
+  reusePreviousTranscript = true,
+  chunkTime = 30
+): Promise<string[]> {
+  const parsed = path.parse(filePath);
+  const outputPath = `${parsed.dir}/${parsed.name}/transcript.json`;
+
+  const exists = await fileExists(outputPath);
+  if (exists && reusePreviousTranscript) {
+    console.log(`Transcription ${outputPath} already exists, skipping`);
+    const fileContent = await readFile(outputPath, "utf8");
+    return outputPath.endsWith("txt") ? fileContent : JSON.parse(fileContent);
+  }
+
+  const chunks = await Downloader.chunk(
+    filePath,
+    parsed.dir,
+    chunkTime,
+    reusePreviousTranscript
+  );
+  const transcription = await Downloader.transcribeChunks(
+    chunks,
+    outputPath,
+    reusePreviousTranscript
+  );
+
+  return transcription;
+}
+
+export async function convertAudioToText(
+  filePath: string,
+  reusePreviousTranscript = true,
+  chunkTime = 30
+) {
+  const audios = await processAudio(
+    filePath,
+    reusePreviousTranscript,
+    chunkTime
+  );
+
+  let fullString = "";
+
+  for (let i = 0; i < audios.length; i++) {
+    const audio = audios[i];
+    fullString += `[${i * chunkTime}:${(i + 1) * chunkTime}s] ${audio}`;
+  }
+
+  return fullString;
+}
+
+export async function processVideo(
+  filePath: string,
+  reusePreviousTranscript = true,
+  chunkTime = 30
+) {
+  const parsed = path.parse(filePath);
+  const outputPath = `${parsed.dir}/${parsed.name}/video.json`;
+
+  console.log("Processing audio...");
+  const transcriptions = await processAudio(
+    filePath,
+    reusePreviousTranscript,
+    chunkTime
+  );
+
+  console.log("Extracting keyframes...");
+  const videoAnalysis = await Downloader.extractKeyframes(
+    filePath,
+    outputPath,
+    chunkTime
+  );
+
+  return videoAnalysis.map((frame, index) => {
+    return {
+      frame,
+      transcription: transcriptions[index],
+    };
+  });
+}
+
+async function convertVideoToText(
+  filePath: string,
+  reusePreviousTranscript = true,
+  chunkTime = 30
+) {
+  const processed = await processVideo(
+    filePath,
+    reusePreviousTranscript,
+    chunkTime
+  );
+
+  let fullString = "";
+
+  for (let i = 0; i < processed.length; i++) {
+    const chunk = processed[i];
+    fullString += `
+    Chunk: (${i + 1}/ ${processed.length}):
+    Start Timestamp: [${i * chunkTime}s]
+    Visual: ${chunk.frame.description}
+    Audio: ${chunk.transcription}
+    End Timestamp: [${i * chunkTime}s]
+    `;
+  }
+
+  return fullString;
+}
+
+async function convertPdfToText(filePath: string) {
+  const existingPdfBytes = fs.readFileSync(filePath);
+  const data = await pdf(existingPdfBytes);
+  return data.text;
+}
+
+export async function convertToText(filePath: string) {
+  const extension = filePath.split(".").pop();
+
+  switch (extension) {
+    case "mp4":
+    case "webm":
+    case "mov":
+    case "mpeg":
+      return convertVideoToText(filePath);
+    case "mp3":
+    case "mpga":
+    case "m4a":
+    case "wav":
+      return convertAudioToText(filePath);
+    case "pdf":
+      return convertPdfToText(filePath);
+    default:
+      return readFile(filePath, "utf8");
+  }
+}

package/src/dataset/diffs/README.md

@@ -0,0 +1,12 @@
+# Goal
+The purpose of this folder is to collect examples of file modifications to fine tune GPT
+
+The dataset should be a JSON reprensentation of a chat thread, where the user has requested some modification to a file, and then a before file and after file contents, which will be fed into the createPatchFile function of the diff tool
+
+The function that generates this structure should be generic so that you can fine tune using examples generated from any selected files in the codebase
+
+## Procedure for generating dataset
+- use glob to list out files that would be in the embeddings
+- for each file, get the commit hash via git log --pretty=oneline -- filepath
+- for each file diff the current text against a git show commit:filepath, so that we can generate a diff of all the previous changes
+- summarize the diff with GPT and have it construct a synthetic user message that would be requesting it to modify the file in those ways

package/src/dataset/diffs/debug-errors.ts

@@ -0,0 +1,29 @@
+import "source-map-support/register";
+
+import { parsePatch, diffLines, applyPatch } from "diff";
+import { PatchingAgent } from "../../agents/patcher/patcher";
+import { ask, writeFile, mkdir, readFile } from "../../utils";
+import { fixPatch, parseHunks } from "../../agents/tools/patch";
+
+async function debugFailure() {
+  const errors = [];
+  for (const data of errors) {
+    try {
+      const { fileContent, fixedPatch, originalPatch } = data;
+      const newFixedPatch = fixPatch(fileContent, originalPatch);
+      console.log({ originalPatch, newFixedPatch });
+      const updatedContent = applyPatch(fileContent, newFixedPatch);
+      console.log("DID PATCH APPLY?", !!updatedContent);
+
+      await ask("See output? ");
+      console.log(updatedContent);
+      await ask("Continue? ");
+    } catch (e) {
+      console.log(e);
+    }
+  }
+}
+
+if (require.main === module) {
+  debugFailure();
+}

package/src/dataset/diffs/debug.ts

@@ -0,0 +1,64 @@
+import "source-map-support/register";
+
+import {
+  ChatCompletionMessageParam,
+  ChatCompletionToolMessageParam,
+  ChatCompletionMessageToolCall,
+} from "openai/resources/chat";
+import { parsePatch, diffLines, applyPatch } from "diff";
+import { PatchingAgent } from "../../agents/patcher/patcher";
+import { ask, writeFile, mkdir, readFile } from "../../utils";
+import { fixPatch, parseHunks } from "../../agents/tools/patch";
+import { md5Hash } from "../../hashes";
+
+async function debugFailure() {
+  const successCount = 0;
+  const attempts = 0;
+  const testHash = "40eda1dc8f1e3052997f2930ae59fed2";
+  const dataset = [];
+
+  let patchData;
+  for (const data of dataset) {
+    if ((await md5Hash(data.patch)) === testHash) {
+      patchData = data;
+      break;
+    }
+  }
+
+  if (patchData) {
+    const originalContent = patchData.before;
+    const dirName = `./src/dataset/diffs/test_files/fail/${testHash}`;
+    const patch = await readFile(`${dirName}/ai-patch.diff`, "utf-8");
+    const fixedPatch = fixPatch(patchData.before, patch);
+    await writeFile(`${dirName}/ai-fixed-patch.diff`, fixedPatch);
+
+    const updatedContent = applyPatch(originalContent, fixedPatch);
+    console.log({ updatedContent });
+    const success = !!updatedContent;
+
+    if (updatedContent === patchData.after || fixedPatch === patchData.patch) {
+      console.log("SUCCESS");
+      console.log("EXACT MATCH");
+    } else {
+      console.log("AI SUGGESTED");
+      console.log(patch);
+
+      console.log("FIXED PATCH");
+      console.log(fixedPatch);
+
+      console.log("ACTUAL");
+      console.log(patchData.patch);
+
+      console.log("DID PATCH APPLY?", !!updatedContent);
+    }
+  }
+
+  const parsedAnswer = parsePatch(patchData.patch);
+  const fileName = parsedAnswer[0].oldFileName;
+
+  console.log({ successCount, attempts, total: dataset.length });
+}
+
+if (require.main === module) {
+  debugFailure();
+}

package/src/dataset/diffs/generate.ts

@@ -0,0 +1,71 @@
+import "source-map-support/register";
+import * as glob from "glob";
+import { createPatch } from "diff";
+import * as fs from "fs";
+import { execAsync } from "../../utils";
+import { summarizeTexts } from "../../ai";
+
+async function generateDataset() {
+  // Define the path pattern for files to include in the embeddings
+  const files = glob.sync("./**/*.ts", {
+    ignore: ["./node_modules/**", "./dist/**", "./.git/**"],
+  });
+
+  const dataset = [];
+
+  for (const filePath of files) {
+    // Get the most recent commit hash for the file
+    const commandOutput = await execAsync(
+      `git log --pretty=oneline -- ${filePath}`
+    );
+    const commitHashes = commandOutput.stdout.split("\n");
+    for (const commitHash of commitHashes) {
+      const firstSpaceIndex = commitHash.indexOf(" ");
+      const hash = commitHash.slice(0, firstSpaceIndex);
+      if (hash === "") {
+        continue;
+      }
+      // Get the diff between the current file and its state at the commit
+      const { stdout: previousFileContent } = await execAsync(
+        `git show ${hash}:${filePath}`
+      );
+      const currentFileContent = fs.readFileSync(filePath, "utf-8");
+
+      // Create a patch/diff of the before and after
+      const patch = createPatch(
+        filePath,
+        previousFileContent,
+        currentFileContent
+      );
+
+      if (patch.split("\n").length < 10) {
+        console.log(`Skipping ${filePath} as the diff is too small`);
+        continue;
+      }
+
+      console.log(filePath);
+      console.log(patch);
+
+      dataset.push({
+        before: previousFileContent,
+        after: currentFileContent,
+        patch,
+      });
+
+      fs.writeFileSync(
+        "./src/dataset/diffs/dataset.json",
+        JSON.stringify(dataset, null, 2)
+      );
+
+      if (dataset.length > 200) {
+        break;
+      }
+    }
+  }
+
+  console.log(`Dataset of ${dataset.length} examples generated`);
+}
+
+if (require.main === module) {
+  generateDataset();
+}

package/src/dataset/diffs/jsonl.ts

@@ -0,0 +1,54 @@
+import "source-map-support/register";
+import * as glob from "glob";
+import { createPatch } from "diff";
+import * as fs from "fs";
+import { execAsync, readFile, writeFile } from "../../utils";
+import { summarizeTexts } from "../../ai";
+
+async function generateDataset() {
+  const datasetFile = await readFile(__dirname + "/dataset.json");
+  const dataset = JSON.parse(datasetFile.toString());
+
+  // convert to jsonl
+  function toMessage(d) {
+    const { userMessage, before, patch } = d;
+    const fileName = patch.split(" ")[1].split("\n")[0];
+
+    return {
+      messages: [
+        {
+          role: "system",
+          content:
+            "Codebase Agent. You use the tools to read and write code, to help the developer implement features faster. Call final answer once you have finished implementing what is requested. As an agent you will receive multiple rounds of input until you call final answer. You are not able to request feedback from the user, so proceed with your plans and the developer will contact you afterwards if they need more help. After modifying files, you will read them to ensure they look correct before calling final answer. You always check your modifications for syntax errors or bugs. You always make the smallest modifications required to files, rather than outputting the entire file. You think step by step about the blocks of code you're modiyfing",
+        },
+        { role: "user", content: userMessage },
+        {
+          role: "assistant",
+          function_call: {
+            name: "readFile",
+            arguments: JSON.stringify({
+              fileName,
+            }),
+          },
+        },
+        {
+          role: "assistant",
+          function_call: {
+            name: "applyPatchFile",
+            arguments: JSON.stringify({
+              fileName,
+              patch,
+            }),
+          },
+        },
+      ],
+    };
+  }
+  const jsonl = dataset.map((d) => JSON.stringify(toMessage(d))).join("\n");
+  await writeFile(__dirname + "/dataset.jsonl", jsonl);
+  console.log(dataset);
+}
+
+if (require.main === module) {
+  generateDataset();
+}