@tyvm/knowhow 0.0.1

package/src/plugins/notion.ts

@@ -0,0 +1,170 @@
+import { Client } from "@notionhq/client";
+import { Plugin } from "./types";
+import { PageObjectResponse } from "@notionhq/client/build/src/api-endpoints";
+import { Embeddable, MinimalEmbedding } from "../types";
+
+export class NotionPlugin implements Plugin {
+  notionClient: Client;
+
+  constructor() {
+    this.notionClient = new Client({
+      auth: process.env.NOTION_TOKEN,
+    });
+  }
+
+  extractUrls(userPrompt: string): string[] {
+    const notionUrlRegex = /https:\/\/www\.notion\.so\/[^\s]+/g;
+    const matches = userPrompt.match(notionUrlRegex);
+    return matches || [];
+  }
+
+  getIdFromUrl(url: string): string {
+    return url.split("-").pop() || "";
+  }
+
+  findKeyInObject<T>(obj: T, searchKey = "plain_text") {
+    if (!obj) {
+      return null;
+    }
+    const keys = Object.keys(obj);
+    for (const key of keys) {
+      if (key === searchKey) {
+        return obj[key];
+      } else if (typeof obj[key] === "object") {
+        const result = this.findKeyInObject(obj[key], searchKey);
+        if (result) {
+          return result;
+        }
+      }
+    }
+  }
+  async getAllChildBlocks(
+    pageId: string,
+    maxDepth = 1,
+    currentDepth = 1,
+    processed = {}
+  ) {
+    if (processed[pageId] || currentDepth > maxDepth) {
+      return { results: [] };
+    }
+    console.log(
+      `Notion Plugin: Fetching all blocks for page ${pageId} at depth ${currentDepth}`
+    );
+    processed[pageId] = true;
+    const response = await this.notionClient.blocks.children.list({
+      block_id: pageId,
+    });
+
+    let cursor = response.next_cursor;
+    while (cursor) {
+      console.log("Fetching more blocks");
+      const childResponse = await this.notionClient.blocks.children.list({
+        block_id: pageId,
+        start_cursor: cursor,
+      });
+      response.results.push(...childResponse.results);
+      cursor = childResponse.next_cursor;
+    }
+
+    for (const block of response.results) {
+      if ("has_children" in block && block.has_children) {
+        const childBlocks = await this.getAllChildBlocks(
+          block.id,
+          maxDepth,
+          currentDepth + 1,
+          processed
+        );
+        response.results.push(...childBlocks.results);
+      }
+    }
+
+    response.has_more = false;
+    response.next_cursor = null;
+
+    return response;
+  }
+
+  async embed(user_input: string) {
+    const embeddings = new Array<MinimalEmbedding>();
+    const urls = this.extractUrls(user_input);
+    const results = await this.getPagesFromUrls(urls);
+    if (!results) {
+      return embeddings;
+    }
+
+    for (const result of results) {
+      let { page, blocks } = result;
+      const childPages = blocks.results.filter(
+        (b) => "has_children" in b && b.has_children
+      );
+
+      console.log(JSON.stringify(results, null, 2));
+
+      const childBlocks = await Promise.all(
+        childPages.map(async (childPage) => {
+          const blocks = await this.getAllChildBlocks(childPage.id);
+          return { childPage, blocks };
+        })
+      );
+
+      for (const child of childBlocks) {
+        const title =
+          "child_page" in child.childPage && child.childPage.child_page;
+        embeddings.push({
+          id: child.childPage.id,
+          text: JSON.stringify(
+            child.blocks.results.map((b) => this.findKeyInObject(b))
+          ),
+          metadata: {
+            ...title,
+          },
+        });
+      }
+    }
+
+    console.log(JSON.stringify(embeddings, null, 2));
+    return embeddings;
+  }
+
+  async getPageEmbedding(page: PageObjectResponse) {
+    let id = page.id;
+    let content = JSON.stringify(page);
+
+    return { id, content };
+  }
+
+  async getPageFromUrl(url: string) {
+    const pageId = url.split("-").pop();
+    if (pageId) {
+      console.log(`Fetching Notion page ${pageId}`);
+      const page = await this.notionClient.pages.retrieve({ page_id: pageId });
+      const blocks = await this.getAllChildBlocks(page.id);
+      return { page, blocks };
+    }
+    return null;
+  }
+
+  async getPagesFromUrls(urls: string[]) {
+    const pages = await Promise.all(
+      urls.map(async (url) => {
+        return this.getPageFromUrl(url);
+      })
+    );
+    return pages;
+  }
+
+  async call(userPrompt: string): Promise<string> {
+    const urls = this.extractUrls(userPrompt);
+    const pages = await this.getPagesFromUrls(urls);
+    const pagesDataFiltered = pages.filter((page) => page !== null);
+    if (pagesDataFiltered.length === 0) {
+      return "NOTION PLUGIN: No pages found";
+    }
+
+    const markdownPages = pagesDataFiltered
+      .map((page) => `### Page: ${JSON.stringify(page, null, 2)}\n-`)
+      .join("\n\n");
+    console.log(markdownPages);
+    return `NOTION PLUGIN: The following pages were loaded:\n\n${markdownPages}`;
+  }
+}

package/src/plugins/plugins.ts

@@ -0,0 +1,56 @@
+import glob from "glob";
+import { Plugin } from "./types";
+import { VimPlugin } from "./vim";
+import { LanguagePlugin } from "./language";
+import { EmbeddingPlugin } from "./embedding";
+import { GitHubPlugin } from "./github";
+import { AsanaPlugin } from "./asana";
+import { LinearPlugin } from "./linear";
+import { JiraPlugin } from "./jira";
+import { NotionPlugin } from "./notion";
+import { DownloaderPlugin } from "./downloader/plugin";
+import { FigmaPlugin } from "./figma";
+import { UrlPlugin } from "./url";
+
+export class PluginService {
+  plugins = {
+    embeddings: new EmbeddingPlugin(),
+    vim: new VimPlugin(),
+    github: new GitHubPlugin(),
+    asana: new AsanaPlugin(),
+    linear: new LinearPlugin(),
+    jira: new JiraPlugin(),
+    notion: new NotionPlugin(),
+    download: new DownloaderPlugin(),
+    figma: new FigmaPlugin(),
+    language: new LanguagePlugin(this),
+    url: new UrlPlugin(),
+  } as Record<string, Plugin>;
+
+  listPlugins() {
+    return Object.keys(this.plugins);
+  }
+
+  isPlugin(name: string) {
+    return name in this.plugins;
+  }
+
+  registerPlugin(name, plugin: Plugin) {
+    this.plugins[name] = plugin;
+  }
+
+  async callMany(plugins: string[], user_input?: string) {
+    const calls = plugins.map((p) => this.plugins[p].call(user_input));
+    return (await Promise.all(calls)).join("\n\n");
+  }
+
+  async call(kind: string, user_input?: string) {
+    return this.plugins[kind].call(user_input);
+  }
+
+  async embed(kind: string, user_input: string) {
+    return this.plugins[kind].embed(user_input);
+  }
+}
+
+export const Plugins = new PluginService();

package/src/plugins/types.ts

@@ -0,0 +1,5 @@
+import { Embeddable, MinimalEmbedding } from "../types";
+export interface Plugin {
+  call(user_input?: string): Promise<string>;
+  embed(user_input?: string): Promise<MinimalEmbedding[]>;
+}

package/src/plugins/url.ts

@@ -0,0 +1,63 @@
+import { Plugin } from "./types";
+import { MinimalEmbedding } from "../types";
+import axios from "axios";
+import * as cheerio from "cheerio";
+import loadWebpage from "../agents/tools/loadWebpage";
+
+export class UrlPlugin implements Plugin {
+  async embed(userPrompt: string): Promise<MinimalEmbedding[]> {
+    const urls = this.extractUrls(userPrompt);
+    const embeddings = await Promise.all(urls.map(this.fetchAndParseUrl));
+    return embeddings.filter((e): e is MinimalEmbedding => e !== null);
+  }
+
+  extractUrls(userPrompt: string): string[] {
+    const urlRegex = /(https?:\/\/[^\s]+)/g;
+    return userPrompt.match(urlRegex) || [];
+  }
+
+  async fetchAndParseUrl(url: string): Promise<MinimalEmbedding | null> {
+    try {
+      const text = await loadWebpage(url);
+
+      console.log(`URL PLUGIN: Fetched content from ${url}:`, text);
+
+      return {
+        id: url + "-url",
+        text,
+        metadata: { url },
+      };
+    } catch (error) {
+      console.error(`Error fetching or parsing URL ${url}:`, error);
+      return null;
+    }
+  }
+
+  async call(userPrompt: string): Promise<string> {
+    const urls = this.extractUrls(userPrompt);
+    if (urls.length === 0) {
+      return "URL PLUGIN: No URLs detected.";
+    }
+
+    const results = await Promise.all(urls.map(this.fetchAndParseUrl));
+    const validResults = results.filter(
+      (r): r is MinimalEmbedding => r !== null
+    );
+
+    if (validResults.length === 0) {
+      return "URL PLUGIN: Failed to fetch or parse any URLs.";
+    }
+
+    const formattedResults = validResults
+      .map(
+        (result) =>
+          `URL: ${result.metadata.url}\n\nContent:\n${result.text.substring(
+            0,
+            500
+          )}...`
+      )
+      .join("\n\n---\n\n");
+
+    return `URL PLUGIN: Successfully fetched and parsed the following URLs:\n\n${formattedResults}`;
+  }
+}

package/src/plugins/vim.ts

@@ -0,0 +1,73 @@
+import glob from "glob";
+import { readFile, fileExists, fileStat } from "../utils";
+import { Plugin } from "./types";
+
+export class VimPlugin implements Plugin {
+  async embed(userPrompt: string) {
+    return [];
+  }
+
+  async getVimFiles() {
+    const vimFiles = await glob.sync("./**/*.swp", { dot: true });
+    return vimFiles;
+  }
+
+  async getSourcePath(vimPath: string, dotFile = false) {
+    const pathParts = vimPath.split("/");
+    const fileName = pathParts[pathParts.length - 1];
+    const cleanedFileName = fileName.slice(1, fileName.length - 4);
+    const sourcePath = pathParts.slice(0, pathParts.length - 1).join("/");
+    const finalPath = sourcePath + "/" + cleanedFileName;
+    const dotFilePath = sourcePath + "/." + cleanedFileName;
+
+    const finalFileExists = await fileExists(finalPath);
+
+    if (finalFileExists) {
+      return finalPath;
+    }
+
+    const dotFileExists = await fileExists(dotFilePath);
+    if (dotFileExists) {
+      return dotFilePath;
+    }
+
+    return finalPath;
+  }
+
+  async getFileContents(swapFile: string) {
+    const filePath = await this.getSourcePath(swapFile);
+    const exists = await fileExists(filePath);
+    if (!exists) {
+      return { filePath, content: "FILE DOES NOT EXIST" };
+    }
+
+    const stat = await fileStat(filePath);
+    if (stat.isDirectory()) {
+      return { filePath, content: "DIRECTORY" };
+    }
+    if (stat.size > 32000) {
+      console.error(
+        `VIM PLUGIN: File ${filePath} is too large with size ${stat.size}`
+      );
+      return { filePath, content: "FILE TOO LARGE" };
+    }
+
+    console.log(`VIM PLUGIN: Reading file ${filePath}`);
+    const content = await readFile(filePath, "utf8");
+    return { filePath, content };
+  }
+
+  async call() {
+    const vimFiles = await this.getVimFiles();
+    const fileContents = await Promise.all(
+      vimFiles.map((f) => this.getFileContents(f))
+    );
+    if (fileContents.length === 0) {
+      return "VIM PLUGIN: No files open in vim";
+    }
+    return (
+      "VIM PLUGIN: The following files are open in vim: " +
+      JSON.stringify(fileContents)
+    );
+  }
+}

package/src/prompts/BasicCodeDocumenter.ts

@@ -0,0 +1 @@
+export default "Summarize this code, with a focus on dependencies, types, models, classes, comments, and examples. Include snippets of core concepts if relevant. \n\n {text}";

package/src/prompts/BasicProjectDocumenter.ts

@@ -0,0 +1 @@
+export default "Summarize this project documentation, with the goal of creating a useful primer for others to understand the project in a technical way. Focus on how the different pieces of the project interact with each other, design patterns, naming conventions, etc. \n\n {text}";

package/src/prompts/EmbeddingSeachHelper.ts

@@ -0,0 +1 @@
+export default `Given this file, create a short summary of what the file contains, and include a list of questions this file would help answer\n\n{text}`;

package/src/prompts/index.ts

@@ -0,0 +1,9 @@
+import BasicCodeDocumenter from "./BasicCodeDocumenter";
+import BasicProjectDocumenter from "./BasicProjectDocumenter";
+import BasicEmbeddingExplainer from "./EmbeddingSeachHelper";
+
+export const Prompts = [
+  { BasicCodeDocumenter },
+  { BasicProjectDocumenter },
+  { BasicEmbeddingExplainer },
+];

package/src/server/index.ts

@@ -0,0 +1,14 @@
+import express from "express";
+
+const app = express();
+const port = 4545;
+
+app.use((req, res) => {
+  const { path, body, query } = req;
+  console.log({ path, body, query });
+  res.send("Hello World!");
+});
+
+app.listen(port, () => {
+  console.log(`Server started at http://localhost:${port}`);
+});

package/src/services/AgentService.ts

@@ -0,0 +1,98 @@
+import { getConfigSync } from "../config";
+import { IAgent } from "../agents/interface";
+import { ConfigAgent } from "../agents/configurable/ConfigAgent";
+import { Events } from "./EventService";
+import { Tools } from "./Tools";
+
+export class AgentService {
+  private agents: Map<string, IAgent> = new Map();
+
+  constructor() {
+    this.wireUp();
+  }
+
+  public wireUp() {
+    Tools.addTool({
+      type: "function",
+      function: {
+        name: "agentCall",
+        description: `Allows an agent to ask another agent a question. Useful for getting help from agents that are configured for specific goals.
+        ${this.getAgentDescriptions()}`,
+        parameters: {
+          type: "object",
+          positional: true,
+          properties: {
+            agentName: {
+              type: "string",
+              description: `The name of the agent to call. Available agents: ${this.listAgents()}`,
+            },
+            query: {
+              type: "string",
+              description: `The query to send to the agent`,
+            },
+          },
+          required: ["agentName", "query"],
+        },
+      },
+    });
+    Events.on("agents:register", (data) => {
+      console.log(`Agent registered: ${data.name}`);
+      const { name, agent } = data;
+      this.registerAgentByName(name, agent);
+    });
+
+    Events.on("agents:call", (data) => {
+      console.log(`Agent called: ${data.name}`);
+      const { name, query, resolve, reject } = data;
+      this.callAgent(name, query).then(resolve).catch(reject);
+    });
+  }
+
+  public registerAgent(agent: IAgent): void {
+    this.registerAgentByName(agent.name, agent);
+  }
+
+  public registerAgentByName(name: string, agent: IAgent): void {
+    this.agents.set(name, agent);
+  }
+
+  public getAgent(name: string): IAgent {
+    const agent = this.agents.get(name);
+    if (!agent) {
+      throw new Error(
+        `Agent ${name} not found. Options are: ${this.listAgents()}`
+      );
+    }
+    return agent;
+  }
+
+  public listAgents(): string[] {
+    return Array.from(this.agents.keys());
+  }
+
+  public getAgentDescriptions() {
+    return Object.keys(this.agents).map((key) => {
+      const agent = this.getAgent(key);
+      return `name: ${agent.name} \n description: ${agent.description}`;
+    });
+  }
+
+  public loadAgentsFromConfig() {
+    const config = getConfigSync();
+    const agents = config.agents || [];
+
+    for (const agent of agents) {
+      this.registerAgent(new ConfigAgent(agent));
+    }
+  }
+
+  public async callAgent(name: string, query: string): Promise<string> {
+    const agent = this.agents.get(name);
+    if (!agent) {
+      return `Agent ${name} not found. Options are: ${this.listAgents()}`;
+    }
+    return agent.call(query);
+  }
+}
+
+export const Agents = new AgentService();