@tyvm/knowhow 0.0.33 → 0.0.34

package/src/clients/knowhowMcp.ts

@@ -0,0 +1,118 @@
+import {
+  GenericClient,
+  CompletionOptions,
+  CompletionResponse,
+  EmbeddingOptions,
+  EmbeddingResponse,
+} from "./types";
+import { McpService } from "../services/Mcp";
+
+/*
+ *
+ * If an MCP supports the following methods, then we can use it as a generic client.
+ * Once it's a generic client, it can be used by agents, or any other part of the system that leverages Clients
+ */
+export class KnowhowMcpClient implements GenericClient {
+  private mcpService: McpService;
+
+  constructor(mcpService: McpService, private handlesConnection = false) {
+    this.mcpService = mcpService;
+  }
+
+  setKey(key: string) {
+    return;
+  }
+
+  async connect(): Promise<void> {
+    if (this.handlesConnection) {
+      await this.mcpService.connectAll();
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    if (this.handlesConnection) {
+      await this.mcpService.closeTransports();
+    }
+  }
+
+  async callFunction<T>(
+    toolName: string,
+    args: Record<string, any>
+  ): Promise<T> {
+    try {
+      await this.connect();
+      const result = await this.mcpService.callFunction<T>(toolName, args);
+      return result as T;
+    } catch (error) {
+      console.error("Error calling MCP function:", toolName, args);
+      console.error(error);
+    } finally {
+      await this.disconnect();
+    }
+  }
+
+  async createChatCompletion(
+    options: CompletionOptions & { provider: string }
+  ): Promise<CompletionResponse> {
+    const data = await this.callFunction<CompletionResponse>(
+      "createAiCompletion",
+      {
+        provider: options.provider || "",
+        options: {
+          ...options,
+          max_tokens: options.max_tokens || 3000,
+
+          ...(options.tools && {
+            tools: options.tools,
+            tool_choice: "auto",
+          }),
+        },
+      }
+    );
+
+    console.log("Completion Response:", JSON.stringify(data, null, 2));
+
+    return data;
+  }
+
+  async createEmbedding(
+    options: EmbeddingOptions & { provider: string }
+  ): Promise<EmbeddingResponse> {
+    return this.callFunction<EmbeddingResponse>("createEmbedding", {
+      provider: options.provider || "",
+      options: {
+        ...options,
+      },
+    });
+  }
+
+  async getModels(): Promise<{ id: string }[]> {
+    try {
+      const parsedResult = await this.callFunction<any>("listAllModels", {});
+      // Convert the models object to the expected format
+      if (typeof parsedResult === "object") {
+        const models: { id: string }[] = [];
+
+        // Handle case where result is already an array of {id: string}
+        if (Array.isArray(parsedResult)) {
+          return parsedResult;
+        }
+
+        // Handle case where result is {provider: [model1, model2]}
+        for (const [provider, modelList] of Object.entries(parsedResult)) {
+          if (Array.isArray(modelList)) {
+            for (const model of modelList) {
+              models.push({ id: `${provider}/${model}` });
+            }
+          }
+        }
+
+        return models;
+      }
+
+      throw new Error("Invalid response format from MCP service");
+    } catch (error) {
+      throw new Error(`Failed to get models via MCP: ${error.message}`);
+    }
+  }
+}

package/src/clients/openai.ts

@@ -17,10 +17,18 @@ import { EmbeddingModels, Models, OpenAiReasoningModels } from "../types";
 
 const config = getConfigSync();
 
-export class GenericOpenAiClient extends OpenAI implements GenericClient {
-  constructor() {
-    super({
-      apiKey: process.env.OPENAI_KEY,
+export class GenericOpenAiClient implements GenericClient {
+  client: OpenAI;
+  apiKey?: string;
+
+  constructor(apiKey = process.env.OPENAI_KEY) {
+    this.setKey(apiKey);
+  }
+
+  setKey(apiKey: string) {
+    this.apiKey = apiKey;
+    this.client = new OpenAI({
+      apiKey,
       ...(config.openaiBaseUrl && { baseURL: config.openaiBaseUrl }),
     });
   }
@@ -40,13 +48,14 @@ export class GenericOpenAiClient extends OpenAI implements GenericClient {
       return msg as ChatCompletionMessageParam;
     });
 
-    const response = await this.chat.completions.create({
+    const response = await this.client.chat.completions.create({
       model: options.model,
       messages: openaiMessages,
       max_tokens: options.max_tokens,
       ...(OpenAiReasoningModels.includes(options.model) && {
         max_tokens: undefined,
-        max_completion_tokens: options.max_tokens,
+        max_completion_tokens: Math.max(options.max_tokens, 100),
+        // Health check requires some thoughts
       }),
 
       ...(options.tools && {
@@ -166,6 +175,21 @@ export class GenericOpenAiClient extends OpenAI implements GenericClient {
         cached_input: 0,
         output: 10.0,
       },
+      [Models.openai.GPT_5]: {
+        input: 1.25,
+        cached_input: 0.125,
+        output: 10,
+      },
+      [Models.openai.GPT_5_Mini]: {
+        input: 0.25,
+        cached_input: 0.025,
+        output: 2,
+      },
+      [Models.openai.GPT_5_Nano]: {
+        input: 0.05,
+        cached_input: 0.005,
+        output: 0.4,
+      },
       /*
        *[Models.openai.Computer_Use]: {
        *  input: 3.0,
@@ -226,7 +250,7 @@ export class GenericOpenAiClient extends OpenAI implements GenericClient {
   }
 
   async getModels() {
-    const models = await this.models.list();
+    const models = await this.client.models.list();
     return models.data.map((m) => {
       return {
         id: m.id,
@@ -237,7 +261,7 @@ export class GenericOpenAiClient extends OpenAI implements GenericClient {
   }
 
   async createEmbedding(options: EmbeddingOptions): Promise<EmbeddingResponse> {
-    const openAiEmbedding = await this.embeddings.create({
+    const openAiEmbedding = await this.client.embeddings.create({
       input: options.input,
       model: options.model,
     });

package/src/clients/types.ts

@@ -85,6 +85,7 @@ export interface EmbeddingResponse {
 }
 
 export interface GenericClient {
+  setKey(key: string): void;
   createChatCompletion(options: CompletionOptions): Promise<CompletionResponse>;
   createEmbedding(options: EmbeddingOptions): Promise<EmbeddingResponse>;
   getModels(): Promise<{ id: string }[]>;

package/src/clients/xai.ts

@@ -16,10 +16,22 @@ import { Models } from "../types";
 
 const config = getConfigSync();
 
-export class GenericXAIClient extends OpenAI implements GenericClient {
-  constructor() {
-    super({
-      apiKey: process.env.XAI_API_KEY,
+export class GenericXAIClient implements GenericClient {
+  private client: OpenAI;
+  private apiKey: string;
+
+  constructor(apiKey = process.env.XAI_API_KEY) {
+    this.apiKey = apiKey || "";
+    this.client = new OpenAI({
+      apiKey: apiKey || process.env.XAI_API_KEY,
+      baseURL: "https://api.x.ai/v1",
+    });
+  }
+
+  setKey(apiKey: string) {
+    this.apiKey = apiKey;
+    this.client = new OpenAI({
+      apiKey,
       baseURL: "https://api.x.ai/v1",
     });
   }
@@ -39,7 +51,7 @@ export class GenericXAIClient extends OpenAI implements GenericClient {
       return msg as ChatCompletionMessageParam;
     });
 
-    const response = await this.chat.completions.create({
+    const response = await this.client.chat.completions.create({
       model: options.model,
       messages: xaiMessages,
       max_tokens: options.max_tokens,

package/src/config.ts

@@ -73,6 +73,14 @@ const defaultConfig = {
   ],
 
   modelProviders: [{ url: "http://localhost:1234", provider: "lms" }],
+
+  ycmd: {
+    enabled: false,
+    installPath: undefined, // Will default to ~/.knowhow/ycmd
+    port: 0, // 0 for auto-assign
+    logLevel: "info",
+    completionTimeout: 5000,
+  },
 } as Config;
 
 const defaultLanguage = {
@@ -148,6 +156,14 @@ export async function updateLanguageConfig(language: Language) {
 }
 
 export async function updateConfig(config: Config) {
+  if (!config || typeof config !== "object") {
+    throw new Error("Invalid config object");
+  }
+
+  await fs.promises.copyFile(
+    ".knowhow/knowhow.json",
+    ".knowhow/knowhow.json.bak"
+  );
   await writeFile(".knowhow/knowhow.json", JSON.stringify(config, null, 2));
 }
 
@@ -185,15 +201,24 @@ export function getConfigSync() {
   }
 }
 
+let loggedWarning = false;
 export async function getConfig() {
   if (!fs.existsSync(".knowhow/knowhow.json")) {
-    console.warn(
-      "KnowHow config file not found. Please run `knowhow init` to create it."
-    );
+    if (!loggedWarning) {
+      loggedWarning = true;
+      console.warn(
+        "KnowHow config file not found. Please run `knowhow init` to create it."
+      );
+    }
     return {} as Config;
   }
-  const config = JSON.parse(await readFile(".knowhow/knowhow.json", "utf8"));
-  return config as Config;
+  try {
+    const config = await readFile(".knowhow/knowhow.json", "utf8");
+    return JSON.parse(config) as Config;
+  } catch (error) {
+    console.error("Error reading .knowhow/knowhow.json:", error);
+    throw new Error("Failed to load KnowHow configuration.");
+  }
 }
 
 export async function loadPrompt(promptName: string) {

package/src/conversion.ts

@@ -2,7 +2,7 @@ import pdf from "pdf-parse";
 import * as fs from "fs";
 import * as path from "path";
 import { readFile, fileExists } from "./utils";
-import { Downloader } from "./plugins/downloader/downloader";
+import { services } from "./services";
 
 export async function processAudio(
   filePath: string,
@@ -21,6 +21,7 @@ export async function processAudio(
       : JSON.parse(fileContent);
   }
 
+  const { Downloader } = services();
   const chunks = await Downloader.chunk(
     filePath,
     parsed.dir,
@@ -73,9 +74,11 @@ export async function processVideo(
   );
 
   console.log("Extracting keyframes...");
+  const { Downloader } = services();
   const videoAnalysis = await Downloader.extractKeyframes(
     filePath,
     outputPath,
+    reusePreviousTranscript,
     chunkTime
   );
 

package/src/login.ts

@@ -3,22 +3,20 @@ import fs from "fs";
 import path from "path";
 import { chmod } from "fs/promises";
 import { ask } from "./utils";
+import { getConfig, updateConfig } from "./config";
+import { KNOWHOW_API_URL } from "./services/KnowhowClient";
 
-const API_URL = process.env.KNOWHOW_API_URL;
-
-export async function login(): Promise<void> {
-  if (!API_URL) {
+export async function login(jwtFlag?: string): Promise<void> {
+  if (!KNOWHOW_API_URL) {
     throw new Error("Error: KNOWHOW_API_URL environment variable not set.");
   }
 
-  const [flag] = process.argv.slice(3);
-
-  if (flag === "--jwt") {
+  if (jwtFlag) {
     const jwt = await ask("Enter your JWT: ");
 
     // Update the JWT file
     const configDir = path.join(process.cwd(), ".knowhow");
-    const jwtFile = path.join(configDir, ".jwt");
+    const jwtFile = path.join(process.cwd(), ".knowhow", ".jwt");
 
     if (!fs.existsSync(configDir)) {
       fs.mkdirSync(configDir, { recursive: true });
@@ -31,7 +29,7 @@ export async function login(): Promise<void> {
   // Get current user/org information
   try {
     const storedJwt = await loadJwt();
-    const response = await axios.get(`${API_URL}/api/users/me`, {
+    const response = await axios.get(`${KNOWHOW_API_URL}/api/users/me`, {
       headers: {
         Authorization: `Bearer ${storedJwt}`,
       },
@@ -47,6 +45,25 @@ export async function login(): Promise<void> {
     console.log(
       `Current user: ${user.email}, \nOrganization: ${currentOrg?.organization?.name} - ${orgId}`
     );
+
+    const config = await getConfig();
+    const proxyUrl = KNOWHOW_API_URL + "/api/proxy";
+    const hasProvider = config.modelProviders.find(
+      (provider) => provider.provider === "knowhow" && provider.url === proxyUrl
+    );
+    if (!hasProvider) {
+      if (!config.modelProviders) {
+        config.modelProviders = [];
+      }
+
+      config.modelProviders.push({
+        provider: "knowhow",
+        url: proxyUrl,
+        jwtFile: ".knowhow/.jwt",
+      });
+
+      await updateConfig(config);
+    }
   } catch (error) {
     if (axios.isAxiosError(error) && error.response) {
       throw new Error(

package/src/microphone.ts

@@ -3,7 +3,6 @@ import * as fs from "fs";
 import * as path from "path";
 import { openai } from "./ai";
 import { execAsync, ask } from "./utils";
-import { Downloader } from "./plugins/downloader/downloader";
 import { convertToText, convertAudioToText } from "./conversion";
 
 interface MicrophoneConfig {

package/src/plugins/downloader/downloader.ts

@@ -5,14 +5,14 @@ import Logger from "progress-estimator";
 import { DownloadInfo, KeyframeInfo, TranscriptChunk } from "./types";
 import { visionTool } from "../../agents/tools/visionTool";
 import { execAsync, fileExists, readFile, mkdir } from "../../utils";
-import { openai } from "../../ai";
+import OpenAI from "openai";
 import { Clients } from "../../clients";
 import { Models } from "../../types";
 
 const logger = Logger();
 
 export class DownloaderService {
-  constructor(private openAi: typeof openai, private clients: typeof Clients) {}
+  constructor(private openAi: OpenAI, private clients: typeof Clients) {}
 
   async askGptVision(
     imageUrl: string,
@@ -99,7 +99,7 @@ export class DownloaderService {
       }
     }
 
-    const command = `ffmpeg -i "${filePath}" -f segment -segment_time ${CHUNK_LENGTH_SECONDS} -map 0:a:0 -acodec mp3 -vn "${outputDirPath}/chunk%03d.mp3"`;
+    const command = `ffmpeg -i "${filePath}" -f segment -segment_time ${CHUNK_LENGTH_SECONDS} -map 0:a:0 -acodec mp3 -vn "${outputDirPath}/chunk%04d.mp3"`;
     await execAsync(command);
 
     const folderFiles = await fs.promises.readdir(outputDirPath);
@@ -115,6 +115,18 @@ export class DownloaderService {
     outputPath: string,
     reusePreviousTranscript = true
   ): AsyncGenerator<TranscriptChunk> {
+    const exists = await fileExists(outputPath);
+    if (exists && reusePreviousTranscript) {
+      console.log("Transcription already exists, using cached data");
+      const contents = await readFile(outputPath);
+      const data = JSON.parse(contents.toString()) as TranscriptChunk[];
+      for (const item of data) {
+        yield item;
+      }
+      return;
+    }
+
+    const allTranscripts = [];
     for (const file of files) {
       const chunkName = path.parse(file).name;
       const chunkTranscriptPath = path.join(
@@ -124,13 +136,19 @@ export class DownloaderService {
       const chunkExists = await fileExists(chunkTranscriptPath);
 
       if (chunkExists && reusePreviousTranscript) {
-        console.log(chunkTranscriptPath, " transcription already exists, skipping");
+        console.log(
+          chunkTranscriptPath,
+          " transcription already exists, using cached data"
+        );
         const contents = await readFile(chunkTranscriptPath);
-        yield {
+        const cached = {
           chunkPath: chunkTranscriptPath,
           text: contents.toString(),
           usd_cost: 0,
         };
+
+        yield cached;
+        allTranscripts.push(cached);
         continue;
       }
 
@@ -149,12 +167,16 @@ export class DownloaderService {
       await fs.promises.writeFile(chunkTranscriptPath, transcript.text);
 
       // save chunk transcript to file
-      yield {
+      const data = {
         chunkPath: chunkTranscriptPath,
         text: transcript.text,
         usd_cost: 30 * 0.0001, // assume 30 seconds,
       };
+      yield data;
+      allTranscripts.push(data);
     }
+
+    fs.writeFileSync(outputPath, JSON.stringify(allTranscripts, null, 2));
   }
 
   public async transcribeChunks(
@@ -164,7 +186,7 @@ export class DownloaderService {
   ): Promise<string[]> {
     const exists = await fileExists(outputPath);
     if (exists && reusePreviousTranscript) {
-      console.log("Transcription already exists, skipping");
+      console.log("Transcription already exists, using cached data");
       const contents = await readFile(outputPath);
       return JSON.parse(contents.toString()) as string[];
     }
@@ -185,12 +207,13 @@ export class DownloaderService {
 
   public async *streamKeyFrameExtraction(
     filePath: string,
-    outputPath: string,
+    videoJsonPath: string,
+    reusePreviousKeyframes: boolean = true,
     interval: number = 10
   ): AsyncGenerator<KeyframeInfo> {
-    if (fs.existsSync(outputPath)) {
-      console.log("Keyframes already exist, skipping");
-      const contents = await readFile(outputPath);
+    if (reusePreviousKeyframes && fs.existsSync(videoJsonPath)) {
+      console.log("Keyframes already exist, using cached data");
+      const contents = await readFile(videoJsonPath);
       const data = JSON.parse(contents.toString()) as KeyframeInfo[];
       for (const keyframe of data) {
         yield { ...keyframe, usd_cost: 0 };
@@ -199,16 +222,18 @@ export class DownloaderService {
     }
 
     const parsed = path.parse(filePath);
-    const outputDir = path.dirname(outputPath);
+    const outputDir = path.dirname(videoJsonPath);
     const fileName = parsed.name;
     const keyframesDir = path.join(outputDir, `/keyframes`);
     await fs.promises.mkdir(keyframesDir, { recursive: true });
 
     const command = `ffmpeg -i "${filePath}" -vf "fps=1/${interval},scale=640:-1" "${keyframesDir}/frame%04d.jpg"`;
     await execAsync(command);
+    console.log("Extracting keyframe:", command);
 
     const keyframes = await fs.promises.readdir(keyframesDir);
 
+    const allKeyframes = [];
     for (const keyframe of keyframes) {
       const keyframePath = path.join(keyframesDir, keyframe);
       const keyframeName = path.parse(keyframe).name;
@@ -218,10 +243,11 @@ export class DownloaderService {
       );
       const descriptionExists = await fileExists(keyframeDescriptionPath);
 
-      if (descriptionExists) {
+      if (descriptionExists && reusePreviousKeyframes) {
         const cached = await readFile(keyframeDescriptionPath);
         const cachedJson = JSON.parse(cached.toString()) as KeyframeInfo;
         yield { ...cachedJson, usd_cost: 0 };
+        allKeyframes.push(cachedJson);
         continue;
       }
 
@@ -237,18 +263,26 @@ export class DownloaderService {
         JSON.stringify(keyframeJson, null, 2)
       );
       yield keyframeJson;
+      allKeyframes.push(keyframeJson);
     }
+
+    await fs.promises.writeFile(
+      videoJsonPath,
+      JSON.stringify(allKeyframes, null, 2)
+    );
   }
 
   public async extractKeyframes(
     filePath: string,
     outputPath: string,
+    reusePreviousKeyframes: boolean = true,
     interval: number = 10
   ): Promise<KeyframeInfo[]> {
     const keyframes: KeyframeInfo[] = [];
     for await (const keyframe of this.streamKeyFrameExtraction(
       filePath,
       outputPath,
+      reusePreviousKeyframes,
       interval
     )) {
       keyframes.push(keyframe);
@@ -265,6 +299,7 @@ export class DownloaderService {
       encoding: "base64",
     });
     const image = `data:image/jpeg;base64,${base64}`;
+    console.log("Describing keyframe:", keyframePath);
     const response = await this.askGptVision(image, question);
     return response;
   }
@@ -285,7 +320,9 @@ export class DownloaderService {
     // Skip chunking if the full output exists
     const exists = await fileExists(outputPath);
     if (exists && reusePreviousTranscript) {
-      console.log(`Transcription ${outputPath} already exists, skipping`);
+      console.log(
+        `Transcription ${outputPath} already exists, using cached data`
+      );
       const fileContent = await readFile(outputPath, "utf8");
       return outputPath.endsWith("txt")
         ? fileContent.split("\n")
@@ -318,14 +355,21 @@ export class DownloaderService {
     // Skip chunking if the full output exists
     const exists = await fileExists(outputPath);
     if (exists && reusePreviousTranscript) {
-      console.log(`Transcription ${outputPath} already exists, skipping`);
+      console.log(
+        `Transcription ${outputPath} already exists, using cached data`
+      );
       const fileContent = await readFile(outputPath, "utf8");
       const lines = outputPath.endsWith("txt")
         ? fileContent.split("\n")
         : JSON.parse(fileContent);
 
-      for (const line of lines)
-        yield { chunkPath: "", text: line, usd_cost: 0 };
+      for (const line of lines) {
+        if (typeof line === "string") {
+          yield { chunkPath: "", text: line, usd_cost: 0 };
+        } else {
+          yield line as TranscriptChunk;
+        }
+      }
       return;
     }
 
@@ -364,6 +408,7 @@ export class DownloaderService {
     const videoAnalysis = await this.extractKeyframes(
       filePath,
       outputPath,
+      reusePreviousTranscript,
       chunkTime
     );
 
@@ -381,19 +426,20 @@ export class DownloaderService {
     chunkTime = 30
   ) {
     const parsed = path.parse(filePath);
-    const outputPath = `${parsed.dir}/${parsed.name}/video.json`;
+    const videoJson = `${parsed.dir}/${parsed.name}/video.json`;
 
     console.log("Processing audio...");
-    const transcriptions = await this.streamProcessAudio(
+    const transcriptions = this.streamProcessAudio(
       filePath,
       reusePreviousTranscript,
       chunkTime
     );
 
     console.log("Extracting keyframes...");
-    const videoAnalysis = await this.streamKeyFrameExtraction(
+    const videoAnalysis = this.streamKeyFrameExtraction(
       filePath,
-      outputPath,
+      videoJson,
+      reusePreviousTranscript,
       chunkTime
     );
 
@@ -402,10 +448,12 @@ export class DownloaderService {
         ?.value as TranscriptChunk;
       yield {
         frame,
-        transcription,
+        transcription: transcription || {
+          chunkPath: "",
+          text: "[missing transcript]",
+          usd_cost: 0,
+        },
       };
     }
   }
 }
-
-export const Downloader = new DownloaderService(openai, Clients);

package/src/plugins/downloader/plugin.ts

@@ -2,7 +2,7 @@ import fs from "fs";
 import { PluginBase, PluginMeta } from "../PluginBase";
 import { MinimalEmbedding } from "../../types";
 import { convertToText, processVideo } from "../../conversion";
-import { Downloader } from "./downloader";
+import { services } from "../../services";
 
 export class DownloaderPlugin extends PluginBase {
   static readonly meta: PluginMeta = {
@@ -40,6 +40,7 @@ export class DownloaderPlugin extends PluginBase {
       try {
         console.log("DOWNLOADER PLUGIN: attempting", url);
         const downloadDir = ".knowhow/downloads/";
+        const { Downloader } = services();
         const fileInfo = await Downloader.download(url, downloadDir);
         const filePath = `${downloadDir}${fileInfo.id}.${fileInfo.ext}`;
         transcript += await convertToText(filePath);
@@ -59,6 +60,7 @@ export class DownloaderPlugin extends PluginBase {
     const embeddings: MinimalEmbedding[] = [];
     for (const url of urls) {
       const downloadDir = ".knowhow/downloads/";
+      const { Downloader } = services();
       const fileInfo = await Downloader.download(url, downloadDir);
       const filePath = `${downloadDir}${fileInfo.id}.${fileInfo.ext}`;
       const processed = await processVideo(filePath);