@tyvm/knowhow 0.0.33 → 0.0.35

package/src/plugins/downloader/downloader.ts

@@ -5,14 +5,14 @@ import Logger from "progress-estimator";
 import { DownloadInfo, KeyframeInfo, TranscriptChunk } from "./types";
 import { visionTool } from "../../agents/tools/visionTool";
 import { execAsync, fileExists, readFile, mkdir } from "../../utils";
-import { openai } from "../../ai";
+import OpenAI from "openai";
 import { Clients } from "../../clients";
 import { Models } from "../../types";
 
 const logger = Logger();
 
 export class DownloaderService {
-  constructor(private openAi: typeof openai, private clients: typeof Clients) {}
+  constructor(private openAi: OpenAI, private clients: typeof Clients) {}
 
   async askGptVision(
     imageUrl: string,
@@ -65,6 +65,36 @@ export class DownloaderService {
     return info;
   }
 
+  private async acquireLock(
+    lockPath: string,
+    timeoutMs = 30000
+  ): Promise<() => void> {
+    const startTime = Date.now();
+
+    while (Date.now() - startTime < timeoutMs) {
+      try {
+        await fs.promises.writeFile(lockPath, process.pid.toString(), {
+          flag: "wx",
+        });
+        console.log(`Lock acquired: ${lockPath}`);
+        return async () => {
+          try {
+            await fs.promises.unlink(lockPath);
+            console.log(`Lock released: ${lockPath}`);
+          } catch (error) {
+            console.warn(`Failed to release lock ${lockPath}:`, error);
+          }
+        };
+      } catch (error) {
+        if (error.code !== "EEXIST") {
+          throw error;
+        }
+        await new Promise((resolve) => setTimeout(resolve, 100));
+      }
+    }
+    throw new Error(`Failed to acquire lock ${lockPath} within ${timeoutMs}ms`);
+  }
+
   public async chunk(
     filePath: string,
     outputDir: string,
@@ -77,44 +107,120 @@ export class DownloaderService {
     console.log({ fileName, fileExt });
     console.log("Chunking file", filePath);
 
-    // create a temp directory
-    const outputDirPath = path.join(outputDir, `${fileName}/chunks`);
-    await fs.promises.mkdir(outputDirPath, { recursive: true });
-    const existingFolderFiles = await fs.promises.readdir(outputDirPath);
-    const existingChunkNames = existingFolderFiles.filter(
-      (f) => f.includes("chunk") && f.endsWith(".mp3")
-    );
+    // Create lock to prevent concurrent chunking of the same file
+    const lockPath = path.join(outputDir, `${fileName}.chunking.lock`);
+    const releaseLock = await this.acquireLock(lockPath);
+
+    try {
+      // create a temp directory
+      const outputDirPath = path.join(outputDir, `${fileName}/chunks`);
+      await fs.promises.mkdir(outputDirPath, { recursive: true });
+      const existingFolderFiles = await fs.promises.readdir(outputDirPath);
+      const existingChunkNames = existingFolderFiles.filter(
+        (f) => f.includes("chunk") && f.endsWith(".mp3")
+      );
 
-    if (existingChunkNames.length > 0) {
-      if (reuseExistingChunks) {
-        console.log("Chunks already exist, skipping");
-        const names = existingChunkNames.map((chunkName) =>
-          path.join(outputDirPath, chunkName)
-        );
-        return names;
-      } else {
-        for (const file of existingFolderFiles) {
-          fs.rmSync(path.join(outputDirPath, file), { recursive: true });
+      if (existingChunkNames.length > 0) {
+        if (reuseExistingChunks) {
+          console.log("Chunks already exist, skipping");
+          await releaseLock();
+          const names = existingChunkNames.map((chunkName) =>
+            path.join(outputDirPath, chunkName)
+          );
+          return names;
+        } else {
+          for (const file of existingFolderFiles) {
+            fs.rmSync(path.join(outputDirPath, file), { recursive: true });
+          }
         }
       }
-    }
 
-    const command = `ffmpeg -i "${filePath}" -f segment -segment_time ${CHUNK_LENGTH_SECONDS} -map 0:a:0 -acodec mp3 -vn "${outputDirPath}/chunk%03d.mp3"`;
-    await execAsync(command);
+      const command = `ffmpeg -i "${filePath}" -f segment -segment_time ${CHUNK_LENGTH_SECONDS} -map 0:a:0 -acodec mp3 -vn "${outputDirPath}/chunk%04d.mp3"`;
+      await execAsync(command);
 
-    const folderFiles = await fs.promises.readdir(outputDirPath);
-    const chunkNames = folderFiles.filter(
-      (f) => f.includes("chunk") && f.endsWith(".mp3")
-    );
-    console.log("Chunked into", chunkNames.length, "chunks");
-    return chunkNames.map((chunkName) => path.join(outputDirPath, chunkName));
+      const folderFiles = await fs.promises.readdir(outputDirPath);
+      const chunkNames = folderFiles.filter(
+        (f) => f.includes("chunk") && f.endsWith(".mp3")
+      );
+      console.log("Chunked into", chunkNames.length, "chunks");
+      return chunkNames.map((chunkName) => path.join(outputDirPath, chunkName));
+    } catch (error) {
+      console.error("Error during chunking:", error);
+      throw error;
+    } finally {
+      await releaseLock();
+    }
   }
 
+  // Add transcription locking to prevent race conditions
+  private transcriptionLocks = new Map<string, Promise<TranscriptChunk[]>>();
+
   public async *streamTranscription(
     files: string[],
     outputPath: string,
     reusePreviousTranscript = true
   ): AsyncGenerator<TranscriptChunk> {
+    const exists = await fileExists(outputPath);
+    if (exists && reusePreviousTranscript) {
+      console.log("Transcription already exists, using cached data");
+      const contents = await readFile(outputPath);
+      const data = JSON.parse(contents.toString()) as TranscriptChunk[];
+      for (const item of data) {
+        yield item;
+      }
+      return;
+    }
+
+    // Prevent concurrent transcription of the same output file
+    const lockKey = outputPath;
+    if (this.transcriptionLocks.has(lockKey)) {
+      console.log("Waiting for concurrent transcription to complete...");
+      await this.transcriptionLocks.get(lockKey);
+      // After waiting, check again if file exists
+      const existsNow = await fileExists(outputPath);
+      if (existsNow && reusePreviousTranscript) {
+        console.log(
+          "Transcription completed by concurrent process, using cached data"
+        );
+        const contents = await readFile(outputPath);
+        const data = JSON.parse(contents.toString()) as TranscriptChunk[];
+        for (const item of data) {
+          yield item;
+        }
+        return;
+      }
+    }
+
+    // Create a promise for this transcription operation
+    const transcriptionPromise = (async () => {
+      const results: TranscriptChunk[] = [];
+      for await (const chunk of this.performTranscription(
+        files,
+        outputPath,
+        reusePreviousTranscript
+      )) {
+        results.push(chunk);
+      }
+      return results;
+    })();
+    this.transcriptionLocks.set(lockKey, transcriptionPromise);
+
+    try {
+      const results = await transcriptionPromise;
+      for (const chunk of results) {
+        yield chunk;
+      }
+    } finally {
+      this.transcriptionLocks.delete(lockKey);
+    }
+  }
+
+  private async *performTranscription(
+    files: string[],
+    outputPath: string,
+    reusePreviousTranscript: boolean
+  ): AsyncGenerator<TranscriptChunk> {
+    const allTranscripts = [];
     for (const file of files) {
       const chunkName = path.parse(file).name;
       const chunkTranscriptPath = path.join(
@@ -124,13 +230,19 @@ export class DownloaderService {
       const chunkExists = await fileExists(chunkTranscriptPath);
 
       if (chunkExists && reusePreviousTranscript) {
-        console.log(chunkTranscriptPath, " transcription already exists, skipping");
+        console.log(
+          chunkTranscriptPath,
+          " transcription already exists, using cached data"
+        );
         const contents = await readFile(chunkTranscriptPath);
-        yield {
+        const cached = {
           chunkPath: chunkTranscriptPath,
           text: contents.toString(),
           usd_cost: 0,
         };
+
+        yield cached;
+        allTranscripts.push(cached);
         continue;
       }
 
@@ -149,12 +261,16 @@ export class DownloaderService {
       await fs.promises.writeFile(chunkTranscriptPath, transcript.text);
 
       // save chunk transcript to file
-      yield {
+      const data = {
         chunkPath: chunkTranscriptPath,
         text: transcript.text,
         usd_cost: 30 * 0.0001, // assume 30 seconds,
       };
+      yield data;
+      allTranscripts.push(data);
     }
+
+    fs.writeFileSync(outputPath, JSON.stringify(allTranscripts, null, 2));
   }
 
   public async transcribeChunks(
@@ -164,7 +280,7 @@ export class DownloaderService {
   ): Promise<string[]> {
     const exists = await fileExists(outputPath);
     if (exists && reusePreviousTranscript) {
-      console.log("Transcription already exists, skipping");
+      console.log("Transcription already exists, using cached data");
       const contents = await readFile(outputPath);
       return JSON.parse(contents.toString()) as string[];
     }
@@ -185,12 +301,13 @@ export class DownloaderService {
 
   public async *streamKeyFrameExtraction(
     filePath: string,
-    outputPath: string,
+    videoJsonPath: string,
+    reusePreviousKeyframes: boolean = true,
     interval: number = 10
   ): AsyncGenerator<KeyframeInfo> {
-    if (fs.existsSync(outputPath)) {
-      console.log("Keyframes already exist, skipping");
-      const contents = await readFile(outputPath);
+    if (reusePreviousKeyframes && fs.existsSync(videoJsonPath)) {
+      console.log("Keyframes already exist, using cached data");
+      const contents = await readFile(videoJsonPath);
       const data = JSON.parse(contents.toString()) as KeyframeInfo[];
       for (const keyframe of data) {
         yield { ...keyframe, usd_cost: 0 };
@@ -199,16 +316,18 @@ export class DownloaderService {
     }
 
     const parsed = path.parse(filePath);
-    const outputDir = path.dirname(outputPath);
+    const outputDir = path.dirname(videoJsonPath);
     const fileName = parsed.name;
     const keyframesDir = path.join(outputDir, `/keyframes`);
     await fs.promises.mkdir(keyframesDir, { recursive: true });
 
     const command = `ffmpeg -i "${filePath}" -vf "fps=1/${interval},scale=640:-1" "${keyframesDir}/frame%04d.jpg"`;
     await execAsync(command);
+    console.log("Extracting keyframe:", command);
 
     const keyframes = await fs.promises.readdir(keyframesDir);
 
+    const allKeyframes = [];
     for (const keyframe of keyframes) {
       const keyframePath = path.join(keyframesDir, keyframe);
       const keyframeName = path.parse(keyframe).name;
@@ -218,10 +337,11 @@ export class DownloaderService {
       );
       const descriptionExists = await fileExists(keyframeDescriptionPath);
 
-      if (descriptionExists) {
+      if (descriptionExists && reusePreviousKeyframes) {
         const cached = await readFile(keyframeDescriptionPath);
         const cachedJson = JSON.parse(cached.toString()) as KeyframeInfo;
         yield { ...cachedJson, usd_cost: 0 };
+        allKeyframes.push(cachedJson);
         continue;
       }
 
@@ -237,18 +357,26 @@ export class DownloaderService {
         JSON.stringify(keyframeJson, null, 2)
       );
       yield keyframeJson;
+      allKeyframes.push(keyframeJson);
     }
+
+    await fs.promises.writeFile(
+      videoJsonPath,
+      JSON.stringify(allKeyframes, null, 2)
+    );
   }
 
   public async extractKeyframes(
     filePath: string,
     outputPath: string,
+    reusePreviousKeyframes: boolean = true,
     interval: number = 10
   ): Promise<KeyframeInfo[]> {
     const keyframes: KeyframeInfo[] = [];
     for await (const keyframe of this.streamKeyFrameExtraction(
       filePath,
       outputPath,
+      reusePreviousKeyframes,
       interval
     )) {
       keyframes.push(keyframe);
@@ -265,6 +393,7 @@ export class DownloaderService {
       encoding: "base64",
     });
     const image = `data:image/jpeg;base64,${base64}`;
+    console.log("Describing keyframe:", keyframePath);
     const response = await this.askGptVision(image, question);
     return response;
   }
@@ -285,7 +414,9 @@ export class DownloaderService {
     // Skip chunking if the full output exists
     const exists = await fileExists(outputPath);
     if (exists && reusePreviousTranscript) {
-      console.log(`Transcription ${outputPath} already exists, skipping`);
+      console.log(
+        `Transcription ${outputPath} already exists, using cached data`
+      );
       const fileContent = await readFile(outputPath, "utf8");
       return outputPath.endsWith("txt")
         ? fileContent.split("\n")
@@ -318,14 +449,21 @@ export class DownloaderService {
     // Skip chunking if the full output exists
     const exists = await fileExists(outputPath);
     if (exists && reusePreviousTranscript) {
-      console.log(`Transcription ${outputPath} already exists, skipping`);
+      console.log(
+        `Transcription ${outputPath} already exists, using cached data`
+      );
       const fileContent = await readFile(outputPath, "utf8");
       const lines = outputPath.endsWith("txt")
         ? fileContent.split("\n")
         : JSON.parse(fileContent);
 
-      for (const line of lines)
-        yield { chunkPath: "", text: line, usd_cost: 0 };
+      for (const line of lines) {
+        if (typeof line === "string") {
+          yield { chunkPath: "", text: line, usd_cost: 0 };
+        } else {
+          yield line as TranscriptChunk;
+        }
+      }
       return;
     }
 
@@ -364,6 +502,7 @@ export class DownloaderService {
     const videoAnalysis = await this.extractKeyframes(
       filePath,
       outputPath,
+      reusePreviousTranscript,
       chunkTime
     );
 
@@ -381,19 +520,20 @@ export class DownloaderService {
     chunkTime = 30
   ) {
     const parsed = path.parse(filePath);
-    const outputPath = `${parsed.dir}/${parsed.name}/video.json`;
+    const videoJson = `${parsed.dir}/${parsed.name}/video.json`;
 
     console.log("Processing audio...");
-    const transcriptions = await this.streamProcessAudio(
+    const transcriptions = this.streamProcessAudio(
       filePath,
       reusePreviousTranscript,
       chunkTime
     );
 
     console.log("Extracting keyframes...");
-    const videoAnalysis = await this.streamKeyFrameExtraction(
+    const videoAnalysis = this.streamKeyFrameExtraction(
       filePath,
-      outputPath,
+      videoJson,
+      reusePreviousTranscript,
       chunkTime
     );
 
@@ -402,10 +542,12 @@ export class DownloaderService {
         ?.value as TranscriptChunk;
       yield {
         frame,
-        transcription,
+        transcription: transcription || {
+          chunkPath: "",
+          text: "[missing transcript]",
+          usd_cost: 0,
+        },
       };
     }
   }
 }
-
-export const Downloader = new DownloaderService(openai, Clients);

package/src/plugins/downloader/plugin.ts

@@ -2,7 +2,7 @@ import fs from "fs";
 import { PluginBase, PluginMeta } from "../PluginBase";
 import { MinimalEmbedding } from "../../types";
 import { convertToText, processVideo } from "../../conversion";
-import { Downloader } from "./downloader";
+import { services } from "../../services";
 
 export class DownloaderPlugin extends PluginBase {
   static readonly meta: PluginMeta = {
@@ -40,6 +40,7 @@ export class DownloaderPlugin extends PluginBase {
       try {
         console.log("DOWNLOADER PLUGIN: attempting", url);
         const downloadDir = ".knowhow/downloads/";
+        const { Downloader } = services();
         const fileInfo = await Downloader.download(url, downloadDir);
         const filePath = `${downloadDir}${fileInfo.id}.${fileInfo.ext}`;
         transcript += await convertToText(filePath);
@@ -59,6 +60,7 @@ export class DownloaderPlugin extends PluginBase {
     const embeddings: MinimalEmbedding[] = [];
     for (const url of urls) {
       const downloadDir = ".knowhow/downloads/";
+      const { Downloader } = services();
       const fileInfo = await Downloader.download(url, downloadDir);
       const filePath = `${downloadDir}${fileInfo.id}.${fileInfo.ext}`;
       const processed = await processVideo(filePath);

package/src/plugins/plugins.ts

@@ -79,6 +79,9 @@ export class PluginService {
   }
 
   async callMany(plugins: string[], userInput?: string) {
+    if (!plugins || plugins.length === 0) {
+      return "";
+    }
     const calls = plugins.map(async (p) => {
       return this.call(p, userInput);
     });