@tyvm/knowhow 0.0.89 → 0.0.91

package/src/{plugins/downloader/downloader.ts → services/MediaProcessorService.ts}

@@ -1,82 +1,54 @@
 import * as fs from "fs";
 import * as path from "path";
-import ytdl from "youtube-dl-exec";
-import Logger from "progress-estimator";
-import { DownloadInfo, KeyframeInfo, TranscriptChunk } from "./types";
-import { visionTool } from "../../agents/tools/visionTool";
-import { execAsync, fileExists, readFile, mkdir } from "../../utils";
-import { Clients } from "../../clients";
-import { Models } from "../../types";
-
-const logger = Logger();
-
-export class DownloaderService {
-  constructor(private clients: typeof Clients) {}
-
-  async askGptVision(
-    imageUrl: string,
-    question: string,
-    provider = "openai",
-    model = Models.openai.GPT_4o
-  ) {
-    const response = await this.clients.createCompletion(provider, {
-      model,
-      max_tokens: 2500,
-      messages: [
-        {
-          role: "user",
-          content: [
-            { type: "text", text: question },
-            {
-              type: "image_url",
-              image_url: {
-                url: imageUrl,
-              },
-            },
-          ],
-        },
-      ],
-    });
+import { exec } from "child_process";
+import { promisify } from "util";
+import { fileExists, readFile, mkdir } from "../utils";
+import { AIClient } from "../clients";
 
-    return response;
-  }
-
-  async download(url: string, outputDir: string) {
-    const info = await this.info(url);
-    const exists = await fileExists(`${outputDir}/${info.id}.${info.ext}`);
+const execPromise = promisify(exec);
 
-    if (exists) {
-      console.log("File already exists, skipping download");
-      return info;
-    }
+async function execAsync(command: string): Promise<string> {
+  const { stdout, stderr } = await execPromise(command);
+  return stdout + stderr;
+}
 
-    const scrape = ytdl(url, { output: `${outputDir}/%(id)s.%(ext)s` });
-    const result = await logger(scrape, `Obtaining ${url}`);
-    return info;
-  }
+export interface TranscriptChunk {
+  chunkPath: string;
+  text: string;
+  usd_cost: number;
+}
 
-  async info(url: string) {
-    const info = await ytdl(url, {
-      dumpSingleJson: true,
-      noWarnings: true,
-    });
-    console.log(info);
-    return info;
-  }
+export interface KeyframeInfo {
+  path: string;
+  description: string;
+  timestamp: number;
+  usd_cost?: number;
+}
 
+/**
+ * MediaProcessorService handles audio/video processing using:
+ * - ffmpeg (system tool) for chunking audio/video
+ * - OpenAI Whisper API for transcription
+ *
+ * This is part of the core services because microphone recording and
+ * audio-to-text transcription are base CLI features. The DownloaderService
+ * (in @tyvm/knowhow-module-video-downloader) uses this service for the
+ * audio/video processing steps after downloading with ytdl.
+ */
+export class MediaProcessorService {
+  constructor(private clients: any) {}
+
+  /**
+   * Split an audio/video file into fixed-length mp3 chunks using ffmpeg.
+   */
   public async chunk(
     filePath: string,
     outputDir: string,
     CHUNK_LENGTH_SECONDS = 30,
     reuseExistingChunks = true
-  ) {
+  ): Promise<string[]> {
     const parsed = path.parse(filePath);
     const fileName = parsed.name;
-    const fileExt = parsed.ext;
-    console.log({ fileName, fileExt });
-    console.log("Chunking file", filePath);
-
-    // create a temp directory
     const outputDirPath = path.join(outputDir, `${fileName}/chunks`);
     await fs.promises.mkdir(outputDirPath, { recursive: true });
     const doneFilePath = path.join(outputDirPath, ".chunking_done");
@@ -90,10 +62,9 @@ export class DownloaderService {
     if (existingChunkNames.length > 0 && doneFileExists) {
       if (reuseExistingChunks) {
         console.log("Chunks already exist, skipping");
-        const names = existingChunkNames.map((chunkName) =>
+        return existingChunkNames.map((chunkName) =>
           path.join(outputDirPath, chunkName)
         );
-        return names;
       } else {
         for (const file of existingFolderFiles) {
           fs.rmSync(path.join(outputDirPath, file), { recursive: true });
@@ -113,6 +84,9 @@ export class DownloaderService {
     return chunkNames.map((chunkName) => path.join(outputDirPath, chunkName));
   }
 
+  /**
+   * Stream transcription of audio chunks using Whisper.
+   */
   public async *streamTranscription(
     files: string[],
     outputPath: string,
@@ -123,13 +97,11 @@ export class DownloaderService {
       console.log("Transcription already exists, using cached data");
       const contents = await readFile(outputPath);
       const data = JSON.parse(contents.toString()) as TranscriptChunk[];
-      for (const item of data) {
-        yield item;
-      }
+      for (const item of data) yield item;
       return;
     }
 
-    const allTranscripts = [];
+    const allTranscripts: TranscriptChunk[] = [];
     for (const file of files) {
       const chunkName = path.parse(file).name;
       const chunkTranscriptPath = path.join(
@@ -139,17 +111,12 @@ export class DownloaderService {
       const chunkExists = await fileExists(chunkTranscriptPath);
 
       if (chunkExists && reusePreviousTranscript) {
-        console.log(
-          chunkTranscriptPath,
-          " transcription already exists, using cached data"
-        );
         const contents = await readFile(chunkTranscriptPath);
-        const cached = {
+        const cached: TranscriptChunk = {
           chunkPath: chunkTranscriptPath,
           text: contents.toString(),
           usd_cost: 0,
         };
-
         yield cached;
         allTranscripts.push(cached);
         continue;
@@ -163,7 +130,7 @@ export class DownloaderService {
           fileName: path.basename(file),
           model: "whisper-1",
         })
-        .catch((e) => {
+        .catch((e: any) => {
           console.error("Error transcribing", file, e);
           return { text: "" };
         });
@@ -171,11 +138,10 @@ export class DownloaderService {
       await mkdir(path.dirname(chunkTranscriptPath), { recursive: true });
       await fs.promises.writeFile(chunkTranscriptPath, transcript.text);
 
-      // save chunk transcript to file
-      const data = {
+      const data: TranscriptChunk = {
         chunkPath: chunkTranscriptPath,
         text: transcript.text,
-        usd_cost: 30 * 0.0001, // assume 30 seconds,
+        usd_cost: 30 * 0.0001,
       };
       yield data;
       allTranscripts.push(data);
@@ -184,6 +150,9 @@ export class DownloaderService {
     fs.writeFileSync(outputPath, JSON.stringify(allTranscripts, null, 2));
   }
 
+  /**
+   * Transcribe all audio chunks and return the text strings.
+   */
   public async transcribeChunks(
     files: string[],
     outputPath: string,
@@ -191,18 +160,16 @@ export class DownloaderService {
   ): Promise<string[]> {
     const exists = await fileExists(outputPath);
     if (exists && reusePreviousTranscript) {
-      console.log("Transcription already exists, using cached data");
       const contents = await readFile(outputPath);
       return JSON.parse(contents.toString()) as string[];
     }
 
-    const fullText = [];
-    for await (const { chunkPath, text } of this.streamTranscription(
+    const fullText: string[] = [];
+    for await (const { text } of this.streamTranscription(
       files,
       outputPath,
       reusePreviousTranscript
     )) {
-      console.log("Chunk transcribed:", chunkPath);
       fullText.push(text);
     }
 
@@ -210,42 +177,65 @@ export class DownloaderService {
     return fullText;
   }
 
+  /**
+   * Process an audio/video file: chunk it with ffmpeg, then transcribe each chunk.
+   * Returns an array of transcript strings (one per chunk).
+   */
+  public async processAudio(
+    filePath: string,
+    reusePreviousTranscript = true,
+    chunkTime = 30
+  ): Promise<string[]> {
+    const parsed = path.parse(filePath);
+    const outputPath = `${parsed.dir}/${parsed.name}/transcript.json`;
+
+    const exists = await fileExists(outputPath);
+    if (exists && reusePreviousTranscript) {
+      const fileContent = (await readFile(outputPath, "utf8")) as string;
+      return outputPath.endsWith("txt")
+        ? fileContent.split("\n")
+        : JSON.parse(fileContent);
+    }
+
+    const chunks = await this.chunk(
+      filePath,
+      parsed.dir,
+      chunkTime,
+      reusePreviousTranscript
+    );
+    return this.transcribeChunks(chunks, outputPath, reusePreviousTranscript);
+  }
+
+  /**
+   * Extract keyframes from a video file using ffmpeg, then describe each with vision AI.
+   */
   public async *streamKeyFrameExtraction(
     filePath: string,
     videoJsonPath: string,
-    reusePreviousKeyframes: boolean = true,
-    interval: number = 10
+    reusePreviousKeyframes = true,
+    interval = 10
   ): AsyncGenerator<KeyframeInfo> {
     if (reusePreviousKeyframes && fs.existsSync(videoJsonPath)) {
-      console.log("Keyframes already exist, using cached data");
       const contents = await readFile(videoJsonPath);
       const data = JSON.parse(contents.toString()) as KeyframeInfo[];
-      for (const keyframe of data) {
-        yield { ...keyframe, usd_cost: 0 };
-      }
+      for (const keyframe of data) yield { ...keyframe, usd_cost: 0 };
       return;
     }
 
-    const parsed = path.parse(filePath);
     const outputDir = path.dirname(videoJsonPath);
-    const fileName = parsed.name;
-    const keyframesDir = path.join(outputDir, `/keyframes`);
+    const keyframesDir = path.join(outputDir, "keyframes");
     await fs.promises.mkdir(keyframesDir, { recursive: true });
 
     const command = `ffmpeg -i "${filePath}" -vf "fps=1/${interval},scale=640:-1" "${keyframesDir}/frame%04d.jpg"`;
     await execAsync(command);
-    console.log("Extracting keyframe:", command);
 
     const keyframes = await fs.promises.readdir(keyframesDir);
+    const allKeyframes: KeyframeInfo[] = [];
 
-    const allKeyframes = [];
     for (const keyframe of keyframes) {
       const keyframePath = path.join(keyframesDir, keyframe);
       const keyframeName = path.parse(keyframe).name;
-      const keyframeDescriptionPath = path.join(
-        keyframesDir,
-        `${keyframeName}.json`
-      );
+      const keyframeDescriptionPath = path.join(keyframesDir, `${keyframeName}.json`);
       const descriptionExists = await fileExists(keyframeDescriptionPath);
 
       if (descriptionExists && reusePreviousKeyframes) {
@@ -257,10 +247,11 @@ export class DownloaderService {
       }
 
       const description = await this.describeKeyframe(keyframePath);
-      const keyframeJson = {
+      const frameNumber = parseInt(keyframe.match(/\d+/)?.[0] ?? "0", 10);
+      const keyframeJson: KeyframeInfo = {
         path: keyframePath,
         description: description.choices[0].message.content,
-        timestamp: this.extractTimestamp(keyframe, interval),
+        timestamp: frameNumber * interval,
         usd_cost: description.usd_cost,
       };
       await fs.promises.writeFile(
@@ -271,17 +262,14 @@ export class DownloaderService {
       allKeyframes.push(keyframeJson);
     }
 
-    await fs.promises.writeFile(
-      videoJsonPath,
-      JSON.stringify(allKeyframes, null, 2)
-    );
+    await fs.promises.writeFile(videoJsonPath, JSON.stringify(allKeyframes, null, 2));
   }
 
   public async extractKeyframes(
     filePath: string,
     outputPath: string,
-    reusePreviousKeyframes: boolean = true,
-    interval: number = 10
+    reusePreviousKeyframes = true,
+    interval = 10
   ): Promise<KeyframeInfo[]> {
     const keyframes: KeyframeInfo[] = [];
     for await (const keyframe of this.streamKeyFrameExtraction(
@@ -292,7 +280,6 @@ export class DownloaderService {
     )) {
       keyframes.push(keyframe);
     }
-
     await fs.promises.writeFile(outputPath, JSON.stringify(keyframes, null, 2));
     return keyframes;
   }
@@ -300,165 +287,20 @@ export class DownloaderService {
   private async describeKeyframe(keyframePath: string) {
     const question =
       "Describe this image in detail, focusing on the main elements and actions visible.";
-    const base64 = await fs.promises.readFile(keyframePath, {
-      encoding: "base64",
-    });
+    const base64 = await fs.promises.readFile(keyframePath, { encoding: "base64" });
     const image = `data:image/jpeg;base64,${base64}`;
-    console.log("Describing keyframe:", keyframePath);
-    const response = await this.askGptVision(image, question);
-    return response;
-  }
-
-  private extractTimestamp(keyframeName: string, interval: number): number {
-    const frameNumber = parseInt(keyframeName.match(/\d+/)[0], 10);
-    return frameNumber * interval;
-  }
-
-  async processAudio(
-    filePath: string,
-    reusePreviousTranscript = true,
-    chunkTime = 30
-  ): Promise<string[]> {
-    const parsed = path.parse(filePath);
-    const outputPath = `${parsed.dir}/${parsed.name}/transcript.json`;
-
-    // Skip chunking if the full output exists
-    const exists = await fileExists(outputPath);
-    if (exists && reusePreviousTranscript) {
-      console.log(
-        `Transcription ${outputPath} already exists, using cached data`
-      );
-      const fileContent = await readFile(outputPath, "utf8");
-      return outputPath.endsWith("txt")
-        ? fileContent.split("\n")
-        : JSON.parse(fileContent);
-    }
-
-    const chunks = await this.chunk(
-      filePath,
-      parsed.dir,
-      chunkTime,
-      reusePreviousTranscript
-    );
-    const transcription = await this.transcribeChunks(
-      chunks,
-      outputPath,
-      reusePreviousTranscript
-    );
-
-    return transcription;
-  }
-
-  async *streamProcessAudio(
-    filePath: string,
-    reusePreviousTranscript = true,
-    chunkTime = 30
-  ): AsyncGenerator<TranscriptChunk> {
-    const parsed = path.parse(filePath);
-    const outputPath = `${parsed.dir}/${parsed.name}/transcript.json`;
-
-    // Skip chunking if the full output exists
-    const exists = await fileExists(outputPath);
-    if (exists && reusePreviousTranscript) {
-      console.log(
-        `Transcription ${outputPath} already exists, using cached data`
-      );
-      const fileContent = await readFile(outputPath, "utf8");
-      const lines = outputPath.endsWith("txt")
-        ? fileContent.split("\n")
-        : JSON.parse(fileContent);
-
-      for (const line of lines) {
-        if (typeof line === "string") {
-          yield { chunkPath: "", text: line, usd_cost: 0 };
-        } else {
-          yield line as TranscriptChunk;
-        }
-      }
-      return;
-    }
-
-    const chunks = await this.chunk(
-      filePath,
-      parsed.dir,
-      chunkTime,
-      reusePreviousTranscript
-    );
-
-    for await (const chunk of this.streamTranscription(
-      chunks,
-      outputPath,
-      reusePreviousTranscript
-    )) {
-      yield chunk;
-    }
-  }
-
-  async processVideo(
-    filePath: string,
-    reusePreviousTranscript = true,
-    chunkTime = 30
-  ) {
-    const parsed = path.parse(filePath);
-    const outputPath = `${parsed.dir}/${parsed.name}/video.json`;
-
-    console.log("Processing audio...");
-    const transcriptions = await this.processAudio(
-      filePath,
-      reusePreviousTranscript,
-      chunkTime
-    );
-
-    console.log("Extracting keyframes...");
-    const videoAnalysis = await this.extractKeyframes(
-      filePath,
-      outputPath,
-      reusePreviousTranscript,
-      chunkTime
-    );
-
-    return videoAnalysis.map((frame, index) => {
-      return {
-        frame,
-        transcription: transcriptions[index],
-      };
-    });
-  }
-
-  async *streamProcessVideo(
-    filePath: string,
-    reusePreviousTranscript = true,
-    chunkTime = 30
-  ) {
-    const parsed = path.parse(filePath);
-    const videoJson = `${parsed.dir}/${parsed.name}/video.json`;
-
-    console.log("Processing audio...");
-    const transcriptions = this.streamProcessAudio(
-      filePath,
-      reusePreviousTranscript,
-      chunkTime
-    );
-
-    console.log("Extracting keyframes...");
-    const videoAnalysis = this.streamKeyFrameExtraction(
-      filePath,
-      videoJson,
-      reusePreviousTranscript,
-      chunkTime
-    );
-
-    for await (const frame of videoAnalysis) {
-      const transcription = (await transcriptions.next())
-        ?.value as TranscriptChunk;
-      yield {
-        frame,
-        transcription: transcription || {
-          chunkPath: "",
-          text: "[missing transcript]",
-          usd_cost: 0,
+    return this.clients.createCompletion("openai", {
+      model: "gpt-4o",
+      max_tokens: 2500,
+      messages: [
+        {
+          role: "user",
+          content: [
+            { type: "text", text: question },
+            { type: "image_url", image_url: { url: image } },
+          ],
         },
-      };
-    }
+      ],
+    });
   }
 }

package/src/services/S3.ts

@@ -12,9 +12,8 @@ import * as fs from "fs";
 const fsPromises = fs.promises;
 import { createWriteStream } from "fs";
 import * as path from "path";
-import { pipeline } from "stream";
+import { pipeline, Readable } from "stream";
 import * as util from "util";
-import axios from "axios";
 
 import { createReadStream } from "fs";
 const pipelineAsync = util.promisify(pipeline);
@@ -83,19 +82,19 @@ export class S3Service {
       const fileStream = createReadStream(filePath);
       const fileStats = await fsPromises.stat(filePath);
 
-      const response = await axios.put(presignedUrl, fileStream, {
-        headers: {
-          "Content-Length": fileStats.size,
-        },
-        maxBodyLength: Infinity,
-        maxContentLength: Infinity,
+      const response = await fetch(presignedUrl, {
+        method: "PUT",
+        headers: { "Content-Length": String(fileStats.size) },
+        body: fileStream as any,
+        // @ts-ignore - Node 18+ supports ReadableStream body with duplex
+        duplex: "half",
       });
 
-      if (response.status === 200) {
-        console.log("File uploaded successfully to pre-signed URL");
-      } else {
+      if (!response.ok) {
         throw new Error(`Upload failed with status code: ${response.status}`);
       }
+
+      console.log("File uploaded successfully to pre-signed URL");
     } catch (error) {
       console.error("Error uploading file to pre-signed URL:", error);
       throw error;
@@ -107,12 +106,14 @@ export class S3Service {
     destinationPath: string
   ): Promise<void> {
     try {
-      const response = await axios.get(presignedUrl, {
-        responseType: "stream",
-      });
+      const response = await fetch(presignedUrl);
+
+      if (!response.ok) {
+        throw new Error(`Download failed with status code: ${response.status}`);
+      }
 
       const fileStream = createWriteStream(destinationPath);
-      await pipelineAsync(response.data, fileStream);
+      await pipelineAsync(Readable.from(response.body as any), fileStream);
 
       console.log(
         `File downloaded successfully from pre-signed URL to ${destinationPath}`
@@ -123,4 +124,3 @@ export class S3Service {
     }
   }
 }
-

package/src/services/index.ts

@@ -1,4 +1,3 @@
-import { DownloaderService } from "../plugins/downloader/downloader";
 import { AIClient, Clients } from "../clients";
 import { AgentService } from "./AgentService";
 import { EventService } from "./EventService";
@@ -14,6 +13,7 @@ import { AgentSyncKnowhowWeb } from "./AgentSyncKnowhowWeb";
 import { AgentSyncFs } from "./AgentSyncFs";
 import { SessionManager } from "./SessionManager";
 import { TaskRegistry } from "./TaskRegistry";
+import { MediaProcessorService } from "./MediaProcessorService";
 
 export * from "./AgentService";
 export * from "./EventService";
@@ -25,6 +25,7 @@ export * from "./LazyToolsService";
 export * as MCP from "./Mcp";
 export * from "./EmbeddingService";
 export * from "./DockerService";
+export * from "./MediaProcessorService";
 export * from "./AgentSyncKnowhowWeb";
 export * from "./AgentSyncFs";
 export * from "./SessionManager";
@@ -45,7 +46,7 @@ let Singletons = {} as {
   knowhowApiClient: KnowhowSimpleClient;
   Plugins: PluginService;
   Clients: AIClient;
-  Downloader: DownloaderService;
+  MediaProcessor: MediaProcessorService;
 };
 
 export const services = (): typeof Singletons => {
@@ -53,7 +54,6 @@ export const services = (): typeof Singletons => {
     const Tools = new ToolsService();
     const Events = new EventService();
     const Agents = new AgentService(Tools, Events);
-    const Downloader = new DownloaderService(Clients);
     const Plugins = new PluginService({
       Agents,
       Events,
@@ -66,11 +66,11 @@ export const services = (): typeof Singletons => {
       AwsS3: new S3Service(),
       Clients,
       Docker: new DockerService(),
-      Downloader,
       Events,
       Flags: new FlagsService(),
       GitHub: new GitHubService(),
       Mcp: new McpService(),
+      MediaProcessor: new MediaProcessorService(Clients),
       Plugins,
       Tools,
       knowhowApiClient: new KnowhowSimpleClient(),

package/src/services/modules/index.ts

@@ -10,12 +10,13 @@ export class ModulesService {
 
     // If no context provided, fall back to global singletons
     if (!context) {
-      const { Clients, Plugins, Agents, Tools } = services();
+      const { Clients, Plugins, Agents, Tools, MediaProcessor } = services();
       context = {
         Agents,
         Plugins,
         Clients,
         Tools,
+        MediaProcessor,
       };
     }
 
@@ -46,7 +47,14 @@ export class ModulesService {
       }
 
       for (const plugin of importedModule.plugins) {
-        pluginService.registerPlugin(plugin.name, plugin.plugin);
+        const pluginContext = {
+          Agents: agentService,
+          Clients: clients,
+          Tools: toolsService,
+          Plugins: pluginService,
+          ...(context.MediaProcessor ? { MediaProcessor: context.MediaProcessor } : {}),
+        };
+        pluginService.registerPlugin(plugin.name, new plugin.plugin(pluginContext as any));
       }
 
       for (const client of importedModule.clients) {