@tyvm/knowhow 0.0.1

package/src/plugins/asana.ts

@@ -0,0 +1,134 @@
+import { Plugin } from "./types";
+import { Embeddable, MinimalEmbedding } from "../types";
+
+export class AsanaPlugin implements Plugin {
+  private asanaClient = require("asana").ApiClient.instance;
+
+  constructor() {
+    this.asanaClient.authentications.token = process.env.ASANA_TOKEN;
+    this.asanaClient.defaultHeaders = {
+      "Asana-Enable": "new_user_task_lists,new_goal_memberships",
+    };
+  }
+
+  getTaskString(task: any) {
+    return `### Task: ${task.name}\n- Description: ${task.notes}\n- URL: ${task.permalink_url}\n Completed: ${task.completed}`;
+  }
+
+  async embed(userPrompt: string): Promise<MinimalEmbedding[]> {
+    const urls = this.extractTaskUrls(userPrompt);
+    const tasksData = await this.getTasksFromUrls(urls);
+    const tasksDataFiltered = tasksData.filter((task) => task !== null);
+
+    const tasksEmbeddings = tasksDataFiltered.map((task) => {
+      return {
+        id: task.permalink_url,
+        text: this.getTaskString(task),
+        metadata: {
+          task: JSON.stringify(task),
+        },
+      };
+    });
+
+    const projectUrls = this.extractProjectUrls(userPrompt);
+    const projectTasks = await this.getTasksFromProjectUrls(projectUrls);
+    const projectTasksFiltered = projectTasks.filter((t) => t !== null);
+
+    const projectTaskEmbeddings = projectTasksFiltered.map((t) => {
+      return {
+        id: t.permalink_url,
+        text: this.getTaskString(t),
+        metadata: {
+          task: JSON.stringify(t),
+        },
+      };
+    });
+
+    const allTasks = tasksEmbeddings.concat(projectTaskEmbeddings);
+    console.log("Found ", allTasks.length, "tasks");
+    return allTasks;
+  }
+
+  extractProjectUrls(userPrompt: string): string[] {
+    const projectUrlRegex = /https:\/\/app\.asana\.com\/0\/(\d+)\/list/g;
+    const matches = userPrompt.match(projectUrlRegex);
+    return matches || [];
+  }
+
+  extractTaskUrls(userPrompt: string): string[] {
+    const taskUrlRegex = /https:\/\/app\.asana\.com\/0\/(\d+)\/(\d+)/g;
+    const matches = userPrompt.match(taskUrlRegex);
+    return matches || [];
+  }
+
+  async getTasksFromUrls(urls: string[]) {
+    const tasks = await Promise.all(
+      urls.map(async (url) => {
+        return this.getTaskFromUrl(url);
+      })
+    );
+    return tasks;
+  }
+
+  async getTasksFromProjectUrls(urls: string[]) {
+    const tasks = await Promise.all(
+      urls.map((url) => this.getTasksFromProjectUrl(url))
+    );
+    return tasks.flat();
+  }
+
+  async getTasksFromProjectUrl(url: string) {
+    const urlParts = url.split("/");
+    const projectId = urlParts[4];
+    console.log({ projectId });
+    if (!projectId) {
+      return null;
+    }
+    let tasks = await this.asanaClient.tasks.findAll({
+      project: projectId,
+      opt_expand: "notes,assignee,permalink_url,custom_fields,tags,completed",
+    });
+    const allData = [];
+
+    do {
+      allData.push(...tasks.data);
+      tasks = await tasks.nextPage();
+    } while (tasks);
+
+    return allData;
+  }
+
+  async getTaskFromUrl(url: string) {
+    const taskId = url.split("/").pop();
+    if (taskId) {
+      console.log(`Fetching Asana task ${taskId}`);
+      return await this.getTaskData(taskId);
+    }
+    return null;
+  }
+
+  async getTaskData(taskId: string) {
+    try {
+      const task = await this.asanaClient.tasks.findById(taskId);
+      return task;
+    } catch (error) {
+      console.error("Error fetching Asana task:", error);
+      return null;
+    }
+  }
+
+  async call(userPrompt: string): Promise<string> {
+    const urls = this.extractTaskUrls(userPrompt);
+    const tasksData = await this.getTasksFromUrls(urls);
+    const tasksDataFiltered = tasksData.filter((task) => task !== null);
+
+    if (tasksDataFiltered.length === 0) {
+      return "ASANA PLUGIN: No tasks found";
+    }
+
+    const markdownTasks = tasksDataFiltered
+      .map((task) => this.getTaskString(task))
+      .join("\n\n");
+    return `ASANA PLUGIN: The following tasks were loaded:\n\n${markdownTasks}`;
+  }
+}

package/src/plugins/downloader/downloader.ts

@@ -0,0 +1,204 @@
+import * as fs from "fs";
+import * as path from "path";
+import ytdl from "youtube-dl-exec";
+import Logger from "progress-estimator";
+import { DownloadInfo, KeyframeInfo } from "./types";
+import { visionTool } from "../../agents/tools/visionTool";
+import { execAsync, fileExists, readFile, mkdir } from "../../utils";
+import { openai } from "../../ai";
+
+const logger = Logger();
+
+class DownloaderService {
+  async download(url: string, outputDir: string) {
+    const info = await this.info(url);
+    const exists = await fileExists(`${outputDir}/${info.id}.${info.ext}`);
+
+    if (exists) {
+      console.log("File already exists, skipping download");
+      return info;
+    }
+
+    const scrape = ytdl(url, { output: `${outputDir}/%(id)s.%(ext)s` });
+    const result = await logger(scrape, `Obtaining ${url}`);
+    return info;
+  }
+
+  async info(url: string) {
+    const info = await ytdl(url, {
+      dumpSingleJson: true,
+      noWarnings: true,
+    });
+    console.log(info);
+    return info;
+  }
+
+  public async chunk(
+    filePath: string,
+    outputDir: string,
+    CHUNK_LENGTH_SECONDS = 30,
+    reuseExistingChunks = true
+  ) {
+    const parsed = path.parse(filePath);
+    const fileName = parsed.name;
+    const fileExt = parsed.ext;
+    console.log({ fileName, fileExt });
+    console.log("Chunking file", filePath);
+
+    // create a temp directory
+    const outputDirPath = path.join(outputDir, `${fileName}/chunks`);
+    await fs.promises.mkdir(outputDirPath, { recursive: true });
+    const existingFolderFiles = await fs.promises.readdir(outputDirPath);
+    const existingChunkNames = existingFolderFiles.filter(
+      (f) => f.includes("chunk") && f.endsWith(".mp3")
+    );
+
+    if (existingChunkNames.length > 0) {
+      if (reuseExistingChunks) {
+        console.log("Chunks already exist, skipping");
+        return existingFolderFiles.map((chunkName) =>
+          path.join(outputDirPath, chunkName)
+        );
+      } else {
+        for (const file of existingFolderFiles) {
+          fs.rmSync(path.join(outputDirPath, file), { recursive: true });
+        }
+      }
+    }
+
+    const command = `ffmpeg -i "${filePath}" -f segment -segment_time ${CHUNK_LENGTH_SECONDS} -map 0:a:0 -acodec mp3 -vn "${outputDirPath}/chunk%03d.mp3"`;
+    await execAsync(command);
+
+    const folderFiles = await fs.promises.readdir(outputDirPath);
+    const chunkNames = folderFiles.filter(
+      (f) => f.includes("chunk") && f.endsWith(".mp3")
+    );
+    console.log("Chunked into", chunkNames.length, "chunks");
+    return chunkNames.map((chunkName) => path.join(outputDirPath, chunkName));
+  }
+
+  public async transcribeChunks(
+    files: string[],
+    outputPath: string,
+    reusePreviousTranscript = true
+  ) {
+    const exists = await fileExists(outputPath);
+    if (exists && reusePreviousTranscript) {
+      console.log("Transcription already exists, skipping");
+      const contents = await readFile(outputPath);
+      return JSON.parse(contents.toString()) as string[];
+    }
+
+    const fullText = [];
+    for (const file of files) {
+      const chunkName = path.parse(file).name;
+      const chunkTranscriptPath = path.join(
+        path.dirname(outputPath),
+        `/chunks/${chunkName}.txt`
+      );
+      const chunkExists = await fileExists(chunkTranscriptPath);
+
+      if (chunkExists && reusePreviousTranscript) {
+        console.log("Chunk transcription already exists, skipping");
+        const contents = await readFile(chunkTranscriptPath);
+        fullText.push(contents.toString());
+        continue;
+      }
+
+      console.log("Transcribing", file);
+      const transcript = await openai.audio.transcriptions
+        .create({
+          file: fs.createReadStream(file),
+          model: "whisper-1",
+        })
+        .catch((e) => {
+          console.error("Error transcribing", file, e);
+          return { text: "" };
+        });
+
+      await mkdir(path.dirname(chunkTranscriptPath), { recursive: true });
+      await fs.promises.writeFile(chunkTranscriptPath, transcript.text);
+
+      // save chunk transcript to file
+      fullText.push(transcript.text);
+    }
+
+    await fs.promises.writeFile(outputPath, JSON.stringify(fullText));
+    return fullText;
+  }
+
+  public async extractKeyframes(
+    filePath: string,
+    outputPath: string,
+    interval: number = 10
+  ): Promise<KeyframeInfo[]> {
+    if (fs.existsSync(outputPath)) {
+      console.log("Keyframes already exist, skipping");
+      const contents = await readFile(outputPath);
+      return JSON.parse(contents.toString()) as KeyframeInfo[];
+    }
+
+    const parsed = path.parse(filePath);
+    const outputDir = path.dirname(outputPath);
+    const fileName = parsed.name;
+    const keyframesDir = path.join(outputDir, `/keyframes`);
+    await fs.promises.mkdir(keyframesDir, { recursive: true });
+
+    const command = `ffmpeg -i "${filePath}" -vf "fps=1/${interval},scale=640:-1" "${keyframesDir}/frame%04d.jpg"`;
+    await execAsync(command);
+
+    const keyframes = await fs.promises.readdir(keyframesDir);
+    const keyframeInfos: KeyframeInfo[] = [];
+
+    for (const keyframe of keyframes) {
+      const keyframePath = path.join(keyframesDir, keyframe);
+      const keyframeName = path.parse(keyframe).name;
+      const keyframeDescriptionPath = path.join(
+        keyframesDir,
+        `${keyframeName}.json`
+      );
+      const descriptionExists = await fileExists(keyframeDescriptionPath);
+
+      if (descriptionExists) {
+        const cached = await readFile(keyframeDescriptionPath);
+        const cachedJson = JSON.parse(cached.toString()) as KeyframeInfo;
+        keyframeInfos.push(cachedJson);
+        continue;
+      }
+
+      const description = await this.describeKeyframe(keyframePath);
+      const keyframeJson = {
+        path: keyframePath,
+        description,
+        timestamp: this.extractTimestamp(keyframe, interval),
+      };
+      await fs.promises.writeFile(
+        keyframeDescriptionPath,
+        JSON.stringify(keyframeJson, null, 2)
+      );
+      keyframeInfos.push(keyframeJson);
+    }
+
+    await fs.promises.writeFile(outputPath, JSON.stringify(keyframeInfos));
+
+    return keyframeInfos;
+  }
+
+  private async describeKeyframe(keyframePath: string): Promise<string> {
+    const question =
+      "Describe this image in detail, focusing on the main elements and actions visible.";
+    const base64 = await fs.promises.readFile(keyframePath, {
+      encoding: "base64",
+    });
+    const image = `data:image/jpeg;base64,${base64}`;
+    const description = await visionTool(image, question);
+    return description;
+  }
+
+  private extractTimestamp(keyframeName: string, interval: number): number {
+    const frameNumber = parseInt(keyframeName.match(/\d+/)[0], 10);
+    return frameNumber * interval;
+  }
+}
+
+export const Downloader = new DownloaderService();

package/src/plugins/downloader/plugin.ts

@@ -0,0 +1,85 @@
+import fs from "fs";
+import { Plugin } from "../types";
+import { MinimalEmbedding } from "../../types";
+import { convertToText, processVideo } from "../../conversion";
+import { Downloader } from "./downloader";
+
+export class DownloaderPlugin implements Plugin {
+  skipExt = ["jpg", "jpeg", "png", "gif"];
+
+  extractUrls(userInput: string): string[] {
+    const urlRegex = /https:\/\/[^\s]+/gim;
+    const matches = userInput.match(urlRegex) || [];
+    return matches;
+  }
+
+  async call(userInput: string): Promise<string> {
+    const urls = this.extractUrls(userInput);
+    if (urls.length === 0) {
+      return "DOWNLOADER PLUGIN: No URLs found in the input";
+    }
+    let transcript = "";
+    for (const url of urls) {
+      if (this.skipExt.includes(url.split(".").pop() || "")) {
+        console.log("DOWNLOADER PLUGIN: skipping", url);
+        continue;
+      }
+      try {
+        console.log("DOWNLOADER PLUGIN: attempting", url);
+        const downloadDir = ".knowhow/downloads/";
+        const fileInfo = await Downloader.download(url, downloadDir);
+        const filePath = `${downloadDir}${fileInfo.id}.${fileInfo.ext}`;
+        transcript += await convertToText(filePath);
+      } catch (e) {
+        console.log("DOWNLOADER PLUGIN: cannot download", url);
+      }
+    }
+    return "DOWNLOADER PLUGIN: " + transcript;
+  }
+
+  async embed(userInput: string): Promise<MinimalEmbedding[]> {
+    const urls = this.extractUrls(userInput);
+    if (urls.length === 0) {
+      return [];
+    }
+
+    const embeddings: MinimalEmbedding[] = [];
+    for (const url of urls) {
+      const downloadDir = ".knowhow/downloads/";
+      const fileInfo = await Downloader.download(url, downloadDir);
+      const filePath = `${downloadDir}${fileInfo.id}.${fileInfo.ext}`;
+      const processed = await processVideo(filePath);
+
+      let index = 0;
+      for (const chunk of processed) {
+        if (chunk.transcription) {
+          embeddings.push({
+            id: `${url}-audio-${index}`,
+            text: chunk.transcription,
+            metadata: {
+              url,
+              description: chunk.frame.description,
+              timestamp: `${chunk.frame.timestamp}s`,
+              image: fs.readFileSync(chunk.frame.path, "base64"),
+            },
+          });
+        }
+
+        if (chunk.frame.description) {
+          embeddings.push({
+            id: `${url}-video-${index}`,
+            text: chunk.frame.description,
+            metadata: {
+              url,
+              timestamp: `${chunk.frame.timestamp}s`,
+            },
+          });
+        }
+
+        index++;
+      }
+    }
+
+    return embeddings;
+  }
+}

package/src/plugins/downloader/types.ts

@@ -0,0 +1,85 @@
+export class AudioFormat {
+  ext: string;
+  width: number | null;
+  height: number | null;
+  tbr: number | null;
+  format_id: string;
+  url: string;
+  vcodec: string;
+  http_headers: Record<string, string>;
+  protocol: string;
+  resolution: string;
+  aspect_ratio: number | null;
+  audio_ext: string;
+  video_ext: string;
+  format: string;
+}
+
+export interface KeyframeInfo {
+  path: string;
+  description: string;
+  timestamp: number;
+}
+
+export class RequestedDownload {
+  ext: string;
+  format_id: string;
+  url: string;
+  vcodec: string;
+  http_headers: Record<string, string>;
+  protocol: string;
+  resolution: string;
+  audio_ext: string;
+  video_ext: string;
+  format: string;
+  epoch: number;
+  _filename: string;
+  __write_download_archive: boolean;
+}
+
+export class DownloadInfo {
+  id: string;
+  title: string;
+  timestamp: Date | null;
+  description: string | null;
+  thumbnail: string | null;
+  age_limit: number;
+  formats: AudioFormat[];
+  subtitles: Record<string, string>;
+  _old_archive_ids: string[];
+  extractor: string;
+  extractor_key: string;
+  webpage_url: string;
+  original_url: string;
+  webpage_url_basename: string;
+  webpage_url_domain: string;
+  playlist: string | null;
+  playlist_index: number | null;
+  display_id: string;
+  fulltitle: string;
+  requested_subtitles: Record<string, string> | null;
+  _has_drm: boolean | null;
+  requested_downloads: RequestedDownload[];
+  ext: string;
+  width: number | null;
+  height: number | null;
+  tbr: number | null;
+  format_id: string;
+  url: string;
+  vcodec: string;
+  http_headers: Record<string, string>;
+  protocol: string;
+  resolution: string;
+  aspect_ratio: number | null;
+  audio_ext: string;
+  video_ext: string;
+  format: string;
+  epoch: number;
+  _type: string;
+  _version: {
+    version: string;
+    current_git_head: string | null;
+    release_git_head: string;
+    repository: string;
+  };
+}

package/src/plugins/embedding.ts

@@ -0,0 +1,42 @@
+import { getConfig } from "../config";
+import {
+  getConfiguredEmbeddings,
+  queryEmbedding,
+  pruneVector,
+  pruneMetadata,
+} from "../embeddings";
+
+import { Plugin } from "./types";
+
+export class EmbeddingPlugin implements Plugin {
+  async embed() {
+    return [];
+  }
+
+  async call(userPrompt: string): Promise<string> {
+    const count = 7;
+    const embeddings = await getConfiguredEmbeddings();
+    const config = await getConfig();
+    const results = await queryEmbedding(
+      userPrompt,
+      embeddings,
+      config.embeddingModel
+    );
+    const context = results.slice(0, count);
+
+    pruneVector(context);
+    pruneMetadata(context);
+
+    for (const entry of context) {
+      console.log(`EMBEDDING PLUGIN: Reading entry ${entry.id}`);
+    }
+
+    const contextLength = JSON.stringify(context).split(" ").length;
+    console.log(
+      `EMBEDDING PLUGIN: Found ${context.length} entries. Loading ${contextLength} words`
+    );
+
+    return `EMBEDDING PLUGIN: Our knowledgebase contains this information which can be used to answer the question:
+    ${JSON.stringify(context)}`;
+  }
+}

package/src/plugins/figma.ts

@@ -0,0 +1,137 @@
+import { Client } from "figma-js";
+import qs from "qs"; // Assumed to be installed
+import { Plugin } from "./types";
+import { MinimalEmbedding } from "../types";
+import { askGptVision } from "../ai";
+
+interface Node {}
+
+interface Component {}
+
+interface ComponentSet {}
+
+interface Style {}
+
+interface FigmaNodeData {
+  document: Node;
+  components: Map<string, Component>;
+  componentSets: Map<string, ComponentSet>;
+  schemaVersion: number;
+  styles: Map<string, Style>;
+}
+
+interface FigmaApiResponse {
+  name: string;
+  role: string;
+  lastModified: string;
+  editorType: string;
+  thumbnailUrl: string;
+  err: string;
+  nodes: Record<string, FigmaNodeData>;
+}
+
+export class FigmaPlugin implements Plugin {
+  private figmaToken: string;
+  private client: ReturnType<typeof Client>;
+
+  constructor() {
+    this.figmaToken = process.env.FIGMA_API_KEY;
+    this.client = Client({ personalAccessToken: this.figmaToken });
+  }
+
+  async loadFigmaData(url: string) {
+    const fileId = this.extractFileIdFromUrl(url);
+    const nodeIds = this.parseNodeIdsFromUrl(url);
+    if (!fileId) {
+      return null;
+    }
+    try {
+      console.log("Fetching figma data", { fileId, nodeIds });
+      const response = await this.client.fileImages(fileId, { ids: nodeIds });
+      return { id: fileId, ...response.data };
+    } catch (error) {
+      console.error("Error fetching Figma file data:", error);
+      return null;
+    }
+  }
+
+  extractFileIdFromUrl(url: string): string | null {
+    const match = /https:\/\/www\.figma\.com\/file\/([A-Za-z0-9]+)/.exec(url);
+    return match ? match[1] : null;
+  }
+
+  // New utility function for parsing node ids from Figma URL
+  parseNodeIdsFromUrl(url: string): string[] | null {
+    const queryIndex = url.indexOf("?");
+    if (queryIndex === -1) return null;
+    const queryStr = url.substring(queryIndex + 1);
+    const queryParams = qs.parse(queryStr);
+    const nodeParam = queryParams["node-id"];
+    let nodeIds = [];
+    if (Array.isArray(nodeParam)) {
+      nodeIds = nodeParam;
+    }
+    if (typeof nodeParam === "string") {
+      nodeIds = nodeParam.split(",");
+    }
+    return nodeIds || null;
+  }
+
+  extractUrls(userPrompt: string): string[] {
+    const urlRegex = /https:\/\/www\.figma\.com\/file\/[^\s]+/g;
+    const matches = userPrompt.match(urlRegex);
+    return matches || [];
+  }
+
+  async embed(userPrompt: string): Promise<MinimalEmbedding[]> {
+    const urls = this.extractUrls(userPrompt);
+    const figmaData = await Promise.all(
+      urls.map((url) => this.loadFigmaData(url))
+    );
+    const filteredData = figmaData.filter((data) => data !== null);
+
+    return filteredData.map((data) => ({
+      id: data.id,
+      text: this.formatFigmaData(data),
+      metadata: {},
+    }));
+  }
+
+  formatFigmaData(data: any): string {
+    return JSON.stringify(data);
+  }
+
+  async call(userPrompt: string): Promise<string> {
+    const urls = this.extractUrls(userPrompt);
+    if (!urls || urls.length === 0) {
+      return "FIGMA PLUGIN: No Figma files found";
+    }
+
+    const figmaData = await Promise.all(
+      urls.map((url) => this.loadFigmaData(url))
+    );
+    const figmaDataFiltered = figmaData.filter((data) => data !== null);
+
+    if (figmaDataFiltered.length === 0) {
+      return "FIGMA PLUGIN: Failed to fetch data for Figma files";
+    }
+
+    const responses = [];
+    for (const data of figmaDataFiltered) {
+      for (const nodeId in data.images) {
+        const imageUrl = data.images[nodeId];
+        const imageDescription = await askGptVision(
+          imageUrl,
+          `Describe the image with relavant information for this user question: ${userPrompt}`
+        );
+
+        console.log("FIGMA PLUGIN: Image description", imageDescription);
+        responses.push({ nodeId, imageDescription });
+      }
+    }
+
+    return `FIGMA PLUGIN: The following Figma files were loaded:\n\n${JSON.stringify(
+      responses
+    )}`;
+  }
+}