@tyvm/knowhow 0.0.90 → 0.0.92

package/src/index.ts

@@ -5,7 +5,6 @@ import {
   getHashes,
   checkNoFilesChanged,
 } from "./hashes";
-import axios from "axios";
 import * as fs from "fs";
 import * as path from "path";
 import * as crypto from "crypto";
@@ -52,6 +51,14 @@ export * as types from "./types";
 export * as processors from "./processors";
 export * as ai from "./ai";
 
+// Export module system types for external modules
+export * from "./services/modules/types";
+// Export plugin types for external plugins
+export { PluginBase } from "./plugins/PluginBase";
+export { PluginMeta, Plugin, PluginContext } from "./plugins/types";
+// Export embedding types
+export { MinimalEmbedding, Embeddable } from "./types";
+
 export async function embed() {
   // load config
   const config = await getConfig();
@@ -98,7 +105,7 @@ export async function purge(globPath: string) {
 
 export async function upload() {
   const config = await getConfig();
-  const { AwsS3, knowhowApiClient } = services();
+  const { AwsS3, Embeddings, knowhowApiClient } = services();
 
   for (const source of config.embedSources) {
     const bucketName = source.remote;
@@ -118,16 +125,15 @@ export async function upload() {
       items,
     };
 
-    if (source.remoteType === "s3") {
+    if (Embeddings.hasResolver(source.remoteType) && source.remoteType !== "knowhow") {
       console.log(
         "Uploading",
         source.output,
         "to",
         `${bucketName}/${embeddingName}.json`
       );
-
-      const s3Key = `${embeddingName}.json`;
-      await AwsS3.uploadFile(source.output, bucketName, s3Key);
+      const remoteKey = `${embeddingName}.json`;
+      await Embeddings.upload(source.remoteType, source.output, bucketName, remoteKey);
     } else if (source.remoteType === "knowhow") {
       if (!source.remoteId) {
         throw new Error("remoteId is required for knowhow uploads");
@@ -350,7 +356,7 @@ export async function handleSingleOutputGeneration(
 
 export async function download() {
   const config = await getConfig();
-  const { AwsS3, GitHub, knowhowApiClient } = services();
+  const { AwsS3, Embeddings, knowhowApiClient } = services();
 
   for (const source of config.embedSources) {
     const { remote, remoteType } = source;
@@ -368,28 +374,17 @@ export async function download() {
     const fileName = `${name}.json`;
     const destinationPath = source.output;
 
-    if (remoteType === "s3") {
-      const bucketName = remote;
+    if (Embeddings.hasResolver(remoteType)) {
       console.log(
         "Downloading",
         fileName,
         `from ${remoteType}`,
-        bucketName,
-        "to",
-        destinationPath
-      );
-      await AwsS3.downloadFile(bucketName, fileName, destinationPath);
-    } else if (remoteType === "github") {
-      console.log(
-        "Downloading",
-        fileName,
-        "from GitHub repo",
         remote,
         "to",
         destinationPath
       );
       const embeddingPath = ".knowhow/embeddings/" + fileName;
-      await GitHub.downloadFile(remote, embeddingPath, destinationPath);
+      await Embeddings.download(remoteType, remote, embeddingPath, destinationPath);
     } else if (remoteType === "knowhow") {
       if (!source.remoteId) {
         throw new Error("remoteId is required for knowhow downloads");

package/src/login.ts

@@ -1,4 +1,4 @@
-import axios from "axios";
+import http from "./utils/http";
 import fs from "fs";
 import path from "path";
 import { chmod } from "fs/promises";
@@ -68,11 +68,10 @@ export async function login(jwtFlag?: boolean): Promise<void> {
       await updateConfig(config);
     }
   } catch (error) {
-    if (axios.isAxiosError(error) && error.response) {
+    if (http.isHttpError(error) && error.response) {
+      const errData = await error.response.json().catch(() => ({ message: "Unknown error" }));
       throw new Error(
-        `Error: ${error.response.status} - ${
-          error.response.data.message || "Unknown error"
-        }`
+        `Error: ${error.status} - ${errData.message || "Unknown error"}`
       );
     }
     console.log(
@@ -98,7 +97,7 @@ export async function loadJwt(): Promise<string> {
 }
 
 export async function checkJwt(storedJwt: string) {
-  const response = await axios.get(`${KNOWHOW_API_URL}/api/users/me`, {
+  const response = await http.get(`${KNOWHOW_API_URL}/api/users/me`, {
     headers: {
       Authorization: `Bearer ${storedJwt}`,
     },

package/src/plugins/language.ts

@@ -6,10 +6,6 @@ import { getConfig, getLanguageConfig } from "../config";
 import { getEnabledPlugins } from "../types";
 import { PluginBase, PluginMeta } from "./PluginBase";
 import { Plugin, PluginContext } from "./types";
-import { GitHubPlugin } from "./github";
-import { AsanaPlugin } from "./asana";
-import { JiraPlugin } from "./jira";
-import { LinearPlugin } from "./linear";
 import { PluginService } from "./plugins";
 
 /**

package/src/plugins/plugins.ts

@@ -4,13 +4,6 @@ import { VimPlugin } from "./vim";
 import { LinterPlugin } from "./LinterPlugin";
 import { LanguagePlugin } from "./language";
 import { EmbeddingPlugin } from "./embedding";
-import { GitHubPlugin } from "./github";
-import { AsanaPlugin } from "./asana";
-import { LinearPlugin } from "./linear";
-import { JiraPlugin } from "./jira";
-import { NotionPlugin } from "./notion";
-import { DownloaderPlugin } from "./downloader/plugin";
-import { FigmaPlugin } from "./figma";
 import { UrlPlugin } from "./url";
 import { GitPlugin } from "./GitPlugin";
 import { TmuxPlugin } from "./tmux";
@@ -33,13 +26,6 @@ export class PluginService {
     this.pluginMap.set("embeddings", new EmbeddingPlugin(context));
     this.pluginMap.set("vim", new VimPlugin(context));
     this.pluginMap.set("linter", new LinterPlugin(context));
-    this.pluginMap.set("github", new GitHubPlugin(context));
-    this.pluginMap.set("asana", new AsanaPlugin(context));
-    this.pluginMap.set("linear", new LinearPlugin(context));
-    this.pluginMap.set("jira", new JiraPlugin(context));
-    this.pluginMap.set("notion", new NotionPlugin(context));
-    this.pluginMap.set("download", new DownloaderPlugin(context));
-    this.pluginMap.set("figma", new FigmaPlugin(context));
     this.pluginMap.set("language", new LanguagePlugin(context));
     this.pluginMap.set("url", new UrlPlugin(context));
     this.pluginMap.set("git", new GitPlugin(context));

package/src/plugins/url.ts

@@ -1,9 +1,6 @@
 import { PluginBase, PluginMeta } from "./PluginBase";
 import { Plugin, PluginContext } from "./types";
 import { MinimalEmbedding } from "../types";
-import axios from "axios";
-import * as cheerio from "cheerio";
-import loadWebpage from "../agents/tools/loadWebpage";
 
 export class UrlPlugin extends PluginBase implements Plugin {
   static readonly meta: PluginMeta = {
@@ -20,22 +17,47 @@ export class UrlPlugin extends PluginBase implements Plugin {
 
   async embed(userPrompt: string): Promise<MinimalEmbedding[]> {
     const urls = this.extractUrls(userPrompt);
-    const embeddings = await Promise.all(urls.map(this.fetchAndParseUrl));
+    const embeddings = await Promise.all(urls.map((url) => this.fetchAndParseUrl(url)));
     return embeddings.filter((e): e is MinimalEmbedding => e !== null);
   }
 
   extractUrls(userPrompt: string): string[] {
     const urlRegex = /(https?:\/\/[^\s]+)/g;
     const urls = userPrompt.match(urlRegex) || [];
-
     return Array.from(new Set(urls));
   }
 
   async fetchAndParseUrl(url: string): Promise<MinimalEmbedding | null> {
     try {
-      const text = await loadWebpage(url);
+      const response = await fetch(url, {
+        headers: {
+          "User-Agent": "Mozilla/5.0 (compatible; KnowhowBot/1.0)",
+          "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        },
+      });
+
+      if (!response.ok) {
+        this.log(`URL PLUGIN: Failed to fetch ${url}: ${response.status}`, "warn");
+        return null;
+      }
+
+      const contentType = response.headers.get("content-type") || "";
+      let text = "";
+
+      if (contentType.includes("text/html") || contentType.includes("text/plain")) {
+        const html = await response.text();
+        // Simple HTML to text: strip tags
+        text = html
+          .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
+          .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
+          .replace(/<[^>]+>/g, " ")
+          .replace(/\s+/g, " ")
+          .trim();
+      } else {
+        text = await response.text();
+      }
 
-      this.log(`URL PLUGIN: Fetched content from ${url}: ${text}`);
+      this.log(`URL PLUGIN: Fetched content from ${url}: ${text.substring(0, 100)}`);
 
       return {
         id: url + "-url",
@@ -58,7 +80,7 @@ export class UrlPlugin extends PluginBase implements Plugin {
       return "URL PLUGIN: Too many URLs detected. Skipping like unintentional bulk browse.";
     }
 
-    const results = await Promise.all(urls.map(this.fetchAndParseUrl));
+    const results = await Promise.all(urls.map((url) => this.fetchAndParseUrl(url)));
     const validResults = results.filter(
       (r): r is MinimalEmbedding => r !== null
     );
@@ -70,10 +92,7 @@ export class UrlPlugin extends PluginBase implements Plugin {
     const formattedResults = validResults
       .map(
         (result) =>
-          `URL: ${result.metadata.url}\n\nContent:\n${result.text.substring(
-            0,
-            500
-          )}...`
+          `URL: ${result.metadata.url}\n\nContent:\n${result.text.substring(0, 500)}...`
       )
       .join("\n\n---\n\n");
 

package/src/services/EmbeddingsService.ts

@@ -0,0 +1,70 @@
+type DownloaderFn = (
+  remote: string,
+  filePath: string,
+  destinationPath: string
+) => Promise<void>;
+
+type UploaderFn = (
+  localPath: string,
+  remote: string,
+  remoteKey: string
+) => Promise<void>;
+
+export interface EmbeddingsResolver {
+  download: DownloaderFn;
+  upload?: UploaderFn;
+}
+
+/**
+ * EmbeddingsService manages pluggable resolvers for remote embedding sources.
+ * Modules can register a resolver for a given remoteType (e.g. "github", "s3")
+ * via registerResolver(), and the core upload()/download() functions delegate to it.
+ */
+export class EmbeddingsService {
+  private resolvers = new Map<string, EmbeddingsResolver>();
+
+  registerResolver(remoteType: string, resolver: EmbeddingsResolver): void {
+    this.resolvers.set(remoteType, resolver);
+  }
+
+  hasResolver(remoteType: string): boolean {
+    return this.resolvers.has(remoteType);
+  }
+
+  async download(
+    remoteType: string,
+    remote: string,
+    filePath: string,
+    destinationPath: string
+  ): Promise<void> {
+    const resolver = this.resolvers.get(remoteType);
+    if (!resolver) {
+      throw new Error(
+        `No resolver registered for remoteType: "${remoteType}". ` +
+          `Install and configure the appropriate knowhow module (e.g. @tyvm/knowhow-module-github).`
+      );
+    }
+    await resolver.download(remote, filePath, destinationPath);
+  }
+
+  async upload(
+    remoteType: string,
+    localPath: string,
+    remote: string,
+    remoteKey: string
+  ): Promise<void> {
+    const resolver = this.resolvers.get(remoteType);
+    if (!resolver) {
+      throw new Error(
+        `No resolver registered for remoteType: "${remoteType}". ` +
+          `Install and configure the appropriate knowhow module (e.g. @tyvm/knowhow-module-s3).`
+      );
+    }
+    if (!resolver.upload) {
+      throw new Error(
+        `Resolver for remoteType: "${remoteType}" does not support uploads.`
+      );
+    }
+    await resolver.upload(localPath, remote, remoteKey);
+  }
+}

package/src/services/KnowhowClient.ts

@@ -1,4 +1,4 @@
-import axios from "axios";
+import http from "../utils/http";
 import fs from "fs";
 import { Message } from "../clients/types";
 import path from "path";
@@ -110,7 +110,7 @@ export const KNOWHOW_API_URL =
   process.env.KNOWHOW_API_URL || "https://api.knowhow.tyvm.ai";
 
 export class KnowhowSimpleClient {
-  headers = {};
+  headers: Record<string, string> = {};
   jwtValidated = false;
 
   constructor(
@@ -156,7 +156,7 @@ export class KnowhowSimpleClient {
       throw new Error("No JWT found. Please login first.");
     }
 
-    return axios.get(`${this.baseUrl}/api/users/me`, {
+    return http.get(`${this.baseUrl}/api/users/me`, {
       headers: this.headers,
     });
   }
@@ -164,7 +164,7 @@ export class KnowhowSimpleClient {
   async getPresignedUploadUrl(source: Config["embedSources"][0]) {
     await this.checkJwt();
     const id = source.remoteId;
-    const presignedUrlResp = await axios.post(
+    const presignedUrlResp = await http.post(
       `${this.baseUrl}/api/org-embeddings/${id}/upload`,
       {},
       {
@@ -181,7 +181,7 @@ export class KnowhowSimpleClient {
   async getPresignedDownloadUrl(source: Config["embedSources"][0]) {
     await this.checkJwt();
     const id = source.remoteId;
-    const presignedUrlResp = await axios.post(
+    const presignedUrlResp = await http.post(
       `${this.baseUrl}/api/org-embeddings/${id}/download`,
       {},
       {
@@ -203,7 +203,7 @@ export class KnowhowSimpleClient {
     }
   ) {
     await this.checkJwt();
-    return axios.put(
+    return http.put(
       `${this.baseUrl}/api/org-embeddings/${id}`,
       data,
       {
@@ -214,7 +214,7 @@ export class KnowhowSimpleClient {
 
   async createChatCompletion(options: CompletionOptions) {
     await this.checkJwt();
-    return axios.post<CompletionResponse>(
+    return http.post<CompletionResponse>(
       `${this.baseUrl}/api/proxy/v1/chat/completions`,
       options,
       {
@@ -225,7 +225,7 @@ export class KnowhowSimpleClient {
 
   async createEmbedding(options: EmbeddingOptions) {
     await this.checkJwt();
-    return axios.post<EmbeddingResponse>(
+    return http.post<EmbeddingResponse>(
       `${this.baseUrl}/api/proxy/v1/embeddings`,
       options,
       {
@@ -236,7 +236,7 @@ export class KnowhowSimpleClient {
 
   async getModels() {
     await this.checkJwt();
-    return axios.get(`${this.baseUrl}/api/proxy/v1/models?type=all`, {
+    return http.get(`${this.baseUrl}/api/proxy/v1/models?type=all`, {
       headers: this.headers,
     });
   }
@@ -262,7 +262,7 @@ export class KnowhowSimpleClient {
     if (options.temperature != null)
       formData.append("temperature", String(options.temperature));
 
-    return axios.post<AudioTranscriptionResponse>(
+    return http.post<AudioTranscriptionResponse>(
       `${this.baseUrl}/api/proxy/v1/audio/transcriptions`,
       formData,
       { headers: { ...this.headers } }
@@ -271,7 +271,7 @@ export class KnowhowSimpleClient {
 
   async createAudioGeneration(options: AudioGenerationOptions) {
     await this.checkJwt();
-    return axios.post<AudioGenerationResponse>(
+    return http.post<AudioGenerationResponse>(
       `${this.baseUrl}/api/proxy/v1/audio/generations`,
       options,
       { headers: this.headers }
@@ -280,7 +280,7 @@ export class KnowhowSimpleClient {
 
   async createImageGeneration(options: ImageGenerationOptions) {
     await this.checkJwt();
-    return axios.post<ImageGenerationResponse>(
+    return http.post<ImageGenerationResponse>(
       `${this.baseUrl}/api/proxy/v1/images/generations`,
       options,
       { headers: this.headers }
@@ -289,7 +289,7 @@ export class KnowhowSimpleClient {
 
   async createVideoGeneration(options: VideoGenerationOptions) {
     await this.checkJwt();
-    return axios.post<VideoGenerationResponse>(
+    return http.post<VideoGenerationResponse>(
       `${this.baseUrl}/api/proxy/v1/videos/generations`,
       options,
       { headers: this.headers }
@@ -299,7 +299,7 @@ export class KnowhowSimpleClient {
   async getVideoStatus(options: VideoStatusOptions) {
     await this.checkJwt();
     const { jobId, ...rest } = options;
-    return axios.get<VideoStatusResponse>(
+    return http.get<VideoStatusResponse>(
       `${this.baseUrl}/api/proxy/v1/videos/${jobId}/status`,
       { headers: this.headers, params: rest }
     );
@@ -308,7 +308,7 @@ export class KnowhowSimpleClient {
   async downloadVideo(options: FileDownloadOptions) {
     await this.checkJwt();
     const { fileId } = options;
-    return axios.get<ArrayBuffer>(
+    return http.get<ArrayBuffer>(
       `${this.baseUrl}/api/proxy/v1/videos/${fileId}/content`,
       { headers: this.headers, responseType: "arraybuffer" }
     );
@@ -323,7 +323,7 @@ export class KnowhowSimpleClient {
       fileName: options.fileName,
       displayName: options.displayName,
     };
-    return axios.post<FileUploadResponse>(
+    return http.post<FileUploadResponse>(
       `${this.baseUrl}/api/proxy/v1/files`,
       body,
       { headers: this.headers }
@@ -333,7 +333,7 @@ export class KnowhowSimpleClient {
   async downloadFile(options: FileDownloadOptions) {
     await this.checkJwt();
     const { fileId } = options;
-    return axios.get<ArrayBuffer>(
+    return http.get<ArrayBuffer>(
       `${this.baseUrl}/api/proxy/v1/files/${fileId}/content`,
       { headers: this.headers, responseType: "arraybuffer" }
     );
@@ -341,7 +341,7 @@ export class KnowhowSimpleClient {
 
   async createChatTask(request: CreateMessageTaskRequest) {
     await this.checkJwt();
-    return axios.post<CreateMessageTaskResponse>(
+    return http.post<CreateMessageTaskResponse>(
       `${this.baseUrl}/api/chat/tasks`,
       request,
       {
@@ -352,7 +352,7 @@ export class KnowhowSimpleClient {
 
   async updateChatTask(taskId: string, updates: UpdateOrgTaskRequest) {
     await this.checkJwt();
-    return axios.put<UpdateOrgTaskResponse>(
+    return http.put<UpdateOrgTaskResponse>(
       `${this.baseUrl}/api/chat/tasks/${taskId}`,
       updates,
       {
@@ -370,7 +370,7 @@ export class KnowhowSimpleClient {
    */
   async getTaskDetails(taskId: string) {
     await this.checkJwt();
-    return axios.get<TaskDetailsResponse>(
+    return http.get<TaskDetailsResponse>(
       `${this.baseUrl}/api/org-agent-tasks/${taskId}`,
       {
         headers: this.headers,
@@ -383,7 +383,7 @@ export class KnowhowSimpleClient {
    */
   async getPendingMessages(taskId: string) {
     await this.checkJwt();
-    return axios.get<PendingMessage[]>(
+    return http.get<PendingMessage[]>(
       `${this.baseUrl}/api/org-agent-tasks/${taskId}/pending-messages`,
       {
         headers: this.headers,
@@ -396,7 +396,7 @@ export class KnowhowSimpleClient {
    */
   async markMessagesAsProcessed(taskId: string, messageIds: string[]) {
     await this.checkJwt();
-    return axios.post<MarkProcessedResponse>(
+    return http.post<MarkProcessedResponse>(
       `${this.baseUrl}/api/org-agent-tasks/${taskId}/pending-messages/mark-processed`,
       { messageIds },
       {
@@ -414,7 +414,7 @@ export class KnowhowSimpleClient {
     role: "user" | "system" = "user"
   ) {
     await this.checkJwt();
-    return axios.post<SendMessageResponse>(
+    return http.post<SendMessageResponse>(
       `${this.baseUrl}/api/org-agent-tasks/${taskId}/messages`,
       { message, role },
       {
@@ -428,7 +428,7 @@ export class KnowhowSimpleClient {
    */
   async pauseAgent(taskId: string) {
     await this.checkJwt();
-    return axios.post<StatusResponse>(
+    return http.post<StatusResponse>(
       `${this.baseUrl}/api/org-agent-tasks/${taskId}/pause`,
       {},
       {
@@ -442,7 +442,7 @@ export class KnowhowSimpleClient {
    */
   async resumeAgent(taskId: string) {
     await this.checkJwt();
-    return axios.post<StatusResponse>(
+    return http.post<StatusResponse>(
       `${this.baseUrl}/api/org-agent-tasks/${taskId}/resume`,
       {},
       {
@@ -464,7 +464,7 @@ export class KnowhowSimpleClient {
    */
   async killAgent(taskId: string) {
     await this.checkJwt();
-    return axios.post<StatusResponse>(
+    return http.post<StatusResponse>(
       `${this.baseUrl}/api/org-agent-tasks/${taskId}/kill`,
       {},
       {
@@ -483,7 +483,7 @@ export class KnowhowSimpleClient {
    */
   async listOrgFiles() {
     await this.checkJwt();
-    return axios.get<
+    return http.get<
       { id: string; fileName: string; folderPath: string; name: string }[]
     >(`${this.baseUrl}/api/org-files`, { headers: this.headers });
   }
@@ -493,7 +493,7 @@ export class KnowhowSimpleClient {
    */
   async createOrgFile(fileName: string, folderPath: string) {
     await this.checkJwt();
-    return axios.post<{
+    return http.post<{
       id: string;
       fileName: string;
       folderPath: string;
@@ -510,7 +510,7 @@ export class KnowhowSimpleClient {
    */
   async getOrgFileText(fileId: string) {
     await this.checkJwt();
-    return axios.get<string>(`${this.baseUrl}/api/org-files/${fileId}/text`, {
+    return http.get<string>(`${this.baseUrl}/api/org-files/${fileId}/text`, {
       headers: this.headers,
       params: { reading: "true" },
     });
@@ -521,7 +521,7 @@ export class KnowhowSimpleClient {
    */
   async updateOrgFileText(fileId: string, text: string) {
     await this.checkJwt();
-    return axios.put(
+    return http.put(
       `${this.baseUrl}/api/org-files/${fileId}/text`,
       { text },
       { headers: this.headers }
@@ -590,7 +590,7 @@ export class KnowhowSimpleClient {
     }
 
     // Get download URL using the file ID
-    const response = await axios.post<{ downloadUrl: string }>(
+    const response = await http.post<{ downloadUrl: string }>(
       `${this.baseUrl}/api/org-files/download/${file.id}`,
       {},
       { headers: this.headers }
@@ -609,7 +609,7 @@ export class KnowhowSimpleClient {
       throw new Error(`File not found: ${filePath}`);
     }
 
-    await axios.post(`${this.baseUrl}/api/org-files/upload/${file.id}/complete`, {}, { headers: this.headers });
+    await http.post(`${this.baseUrl}/api/org-files/upload/${file.id}/complete`, {}, { headers: this.headers });
   }
 
   /**
@@ -627,7 +627,7 @@ export class KnowhowSimpleClient {
     const fileName = lastSlash >= 0 ? filePath.substring(lastSlash + 1) : filePath;
 
     // Get upload URL using the file ID
-    const response = await axios.post<{ uploadUrl: string }>(
+    const response = await http.post<{ uploadUrl: string }>(
       `${this.baseUrl}/api/org-files/upload/${file.id}`,
       { fileName },
       { headers: this.headers }
@@ -641,7 +641,7 @@ export class KnowhowSimpleClient {
    */
   async getGitCredential(repo: string): Promise<GitCredentialResponse> {
     await this.checkJwt();
-    const response = await axios.post<GitCredentialResponse>(
+    const response = await http.post<GitCredentialResponse>(
       `${this.baseUrl}/api/github/git-credential`,
       { repo },
       { headers: this.headers }