@tyvm/knowhow 0.0.109-dev.e88af1e → 0.0.110

package/src/services/MediaProcessorService.ts

@@ -1,9 +1,10 @@
 import * as fs from "fs";
 import * as path from "path";
-import { exec } from "child_process";
+import { exec, spawn } from "child_process";
 import { promisify } from "util";
 import { fileExists, readFile, mkdir } from "../utils";
 import { AIClient } from "../clients";
+import { Models } from "../types";
 
 const execPromise = promisify(exec);
 
@@ -36,7 +37,7 @@ export interface KeyframeInfo {
  * audio/video processing steps after downloading with ytdl.
  */
 export class MediaProcessorService {
-  constructor(private clients: any) {}
+  constructor(private clients: AIClient) {}
 
   /**
    * Split an audio/video file into fixed-length mp3 chunks using ffmpeg.
@@ -45,7 +46,8 @@ export class MediaProcessorService {
     filePath: string,
     outputDir: string,
     CHUNK_LENGTH_SECONDS = 30,
-    reuseExistingChunks = true
+    reuseExistingChunks = true,
+    onProgress?: (progressFraction: number) => void
   ): Promise<string[]> {
     const parsed = path.parse(filePath);
     const fileName = parsed.name;
@@ -72,8 +74,70 @@ export class MediaProcessorService {
       }
     }
 
-    const command = `ffmpeg -i "${filePath}" -f segment -segment_time ${CHUNK_LENGTH_SECONDS} -map 0:a:0 -acodec mp3 -vn "${outputDirPath}/chunk%04d.mp3"`;
-    await execAsync(command);
+    // Use faster encoding settings:
+    // - mono audio (-ac 1): halves encoding work, Whisper handles mono fine
+    // - low bitrate (-b:a 32k): sufficient for speech, much faster encode + smaller files
+    // - fast preset not available for mp3 encoder, but limiting bitrate helps
+    // - -threads 0: use all available CPU threads for faster processing
+    // If the input is already an mp3, copy the audio stream to avoid re-encoding
+    const inputExt = path.extname(filePath).toLowerCase().replace('.', '');
+    const isAlreadyMp3 = inputExt === 'mp3';
+    const audioCodecArgs = isAlreadyMp3
+      ? '-acodec copy'
+      : '-acodec libmp3lame -ac 1 -b:a 32k -threads 0';
+
+    // Use -progress pipe:1 to get real-time progress from ffmpeg
+    // We need the total duration first to calculate fraction
+    await new Promise<void>((resolve, reject) => {
+      // Get total duration via ffprobe first
+      let totalDurationSeconds = 0;
+      exec(
+        `ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "${filePath}"`,
+        (err, stdout) => {
+          if (!err && stdout.trim()) {
+            totalDurationSeconds = parseFloat(stdout.trim()) || 0;
+          }
+
+          // Now run ffmpeg with progress reporting
+          const args = [
+            '-i', filePath,
+            '-f', 'segment',
+            '-segment_time', String(CHUNK_LENGTH_SECONDS),
+            '-map', '0:a:0',
+            ...audioCodecArgs.split(' '),
+            '-vn',
+            ...(onProgress ? ['-progress', 'pipe:1'] : []),
+            `${outputDirPath}/chunk%04d.mp3`,
+          ];
+
+          const proc = spawn('ffmpeg', args);
+
+          let stdoutBuf = '';
+          proc.stdout?.on('data', (data: Buffer) => {
+            stdoutBuf += data.toString();
+            if (onProgress && totalDurationSeconds > 0) {
+              // ffmpeg -progress outputs key=value lines; look for out_time_ms
+              const match = stdoutBuf.match(/out_time_ms=(\d+)/g);
+              if (match) {
+                const last = match[match.length - 1];
+                const ms = parseInt(last.split('=')[1], 10);
+                const fraction = Math.min(ms / 1000 / totalDurationSeconds, 1);
+                onProgress(fraction);
+                // Keep only tail to avoid unbounded buffer growth
+                stdoutBuf = stdoutBuf.slice(-500);
+              }
+            }
+          });
+
+          proc.on('close', (code) => {
+            if (code === 0) resolve();
+            else reject(new Error(`ffmpeg exited with code ${code}`));
+          });
+          proc.on('error', reject);
+        }
+      );
+    });
+
     await fs.promises.writeFile(doneFilePath, "done");
 
     const folderFiles = await fs.promises.readdir(outputDirPath);
@@ -298,8 +362,9 @@ export class MediaProcessorService {
     });
     const image = `data:image/jpeg;base64,${base64}`;
     return this.clients.createCompletion("openai", {
-      model: "gpt-4o",
+      model: Models.openai.GPT_4o,
       max_tokens: 2500,
+      timeout: 20000,
       messages: [
         {
           role: "user",
@@ -315,7 +380,8 @@ export class MediaProcessorService {
   async *streamProcessVideo(
     filePath: string,
     reusePreviousTranscript = true,
-    chunkTime = 30
+    chunkTime = 30,
+    onChunkingProgress?: (fraction: number) => void
   ) {
     const parsed = path.parse(filePath);
     const videoJson = `${parsed.dir}/${parsed.name}/video.json`;
@@ -324,7 +390,8 @@ export class MediaProcessorService {
     const transcriptions = this.streamProcessAudio(
       filePath,
       reusePreviousTranscript,
-      chunkTime
+      chunkTime,
+      onChunkingProgress
     );
 
     console.log("Extracting keyframes...");
@@ -352,7 +419,8 @@ export class MediaProcessorService {
   async *streamProcessAudio(
     filePath: string,
     reusePreviousTranscript = true,
-    chunkTime = 30
+    chunkTime = 30,
+    onChunkingProgress?: (fraction: number) => void
   ): AsyncGenerator<TranscriptChunk> {
     const parsed = path.parse(filePath);
     const outputPath = `${parsed.dir}/${parsed.name}/transcript.json`;
@@ -382,7 +450,8 @@ export class MediaProcessorService {
       filePath,
       parsed.dir,
       chunkTime,
-      reusePreviousTranscript
+      reusePreviousTranscript,
+      onChunkingProgress
     );
 
     for await (const chunk of this.streamTranscription(

package/src/services/modules/index.ts

@@ -31,14 +31,10 @@ export class ModulesService {
     // puts packages), then cwd node_modules, then global node_modules.
     // This allows modules installed via `knowhow modules install` to be found
     // even when knowhow itself is installed globally.
-    const cwdPaths = (require as any).resolve
-      ? require.resolve.paths?.("") || []
-      : [];
     const resolvePaths = [
       path.join(process.cwd(), ".knowhow", "node_modules"),
       path.join(os.homedir(), ".knowhow", "node_modules"),
       path.join(process.cwd(), "node_modules"),
-      ...cwdPaths,
     ];
 
     for (const modulePath of allModulePaths) {
@@ -57,22 +53,31 @@ export class ModulesService {
           resolvedPath = modulePath; // fall back to normal require resolution
         }
       }
-      const rawModule = require(resolvedPath);
-      const importedModule = (rawModule.default || rawModule) as KnowhowModule;
-      context.Events?.log(
-        "ModulesService",
-        `🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`
-      );
-      await importedModule.init({
-        config,
-        cwd: process.cwd(),
-        context: context as ModuleContext,
-      });
-      context.Events?.log(
-        "ModulesService",
-        `✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`
-      );
 
+      let importedModule: KnowhowModule;
+      try {
+        const rawModule = require(resolvedPath);
+        importedModule = (rawModule.default || rawModule) as KnowhowModule;
+        context.Events?.log(
+          "ModulesService",
+          `🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`
+        );
+        await importedModule.init({
+          config,
+          cwd: process.cwd(),
+          context: context as ModuleContext,
+        });
+        context.Events?.log(
+          "ModulesService",
+          `✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`
+        );
+      } catch (err: any) {
+        process.stderr.write(
+          `\n⚠️  Failed to load module "${modulePath}": ${err.message}\n` +
+          `   Run "knowhow modules setup --global" or "knowhow modules install ${modulePath} --global" to fix this.\n\n`
+        );
+        continue;
+      }
       // Only register tools/agents/plugins/clients if the relevant services
       // are available in context (they may not be during early CLI command registration)
       if (context.Agents) {

package/tests/processors/Base64ImageDetector.test.ts

@@ -594,3 +594,163 @@ describe("Base64ImageDetector", () => {
     });
   });
 });
+
+describe("Base64ImageDetector - image path hint detection", () => {
+  let detector: Base64ImageProcessor;
+  let processor: ReturnType<Base64ImageProcessor["createProcessor"]>;
+
+  beforeEach(() => {
+    detector = new Base64ImageProcessor();
+    processor = detector.createProcessor();
+  });
+
+  /**
+   * Simulates the actual output from the Playwright MCP browser_take_screenshot tool.
+   * When an agent calls browser_take_screenshot, the tool message content looks like:
+   *
+   *   ### Result
+   *   - [Screenshot of viewport](./hackernews-screenshot.png)
+   *   ### Ran Playwright code
+   *   ...
+   *
+   * The Base64ImageDetector should detect the .png path in this text and add a hint
+   * telling the model it can call loadImageAsBase64 to actually view the image.
+   */
+  describe("browser screenshot tool response", () => {
+    it("should detect image path from a browser screenshot tool message and add hint", () => {
+      // This is the exact format returned by the browser MCP take_screenshot tool
+      const screenshotToolResponse =
+        "### Result\n- [Screenshot of viewport](./hackernews-screenshot.png)\n### Ran Playwright code\n```js\nawait page.screenshot({ path: './hackernews-screenshot.png', scale: 'css', type: 'png' });\n```";
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: screenshotToolResponse,
+          tool_call_id: "call_abc123",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      const content = modifiedMessages[0].content as string;
+      expect(typeof content).toBe("string");
+      // Should still contain the original text
+      expect(content).toContain("Screenshot of viewport");
+      expect(content).toContain("./hackernews-screenshot.png");
+      // Should contain the hint
+      expect(content).toContain("[TIP:");
+      expect(content).toContain("loadImageAsBase64");
+      expect(content).toContain("./hackernews-screenshot.png");
+    });
+
+    it("should include the exact file path in the hint", () => {
+      const screenshotPath = "./hackernews-screenshot.png";
+      const toolResponse = `### Result\n- [Screenshot of viewport](${screenshotPath})\n`;
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: toolResponse,
+          tool_call_id: "call_xyz789",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      const content = modifiedMessages[0].content as string;
+      // The hint should reference the exact path
+      expect(content).toContain(`loadImageAsBase64("${screenshotPath}")`);
+    });
+
+    it("should not add hint when message contains no image paths", () => {
+      const toolResponse = "The page has been loaded successfully.";
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: toolResponse,
+          tool_call_id: "call_noimages",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      const content = modifiedMessages[0].content as string;
+      expect(content).toBe(toolResponse);
+      expect(content).not.toContain("[TIP:");
+    });
+
+    it("should detect absolute paths like /tmp/page-123.png", () => {
+      const toolResponse =
+        "Screenshot saved to /tmp/page-2026-01-01T12-00-00.png for review.";
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: toolResponse,
+          tool_call_id: "call_abs",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      const content = modifiedMessages[0].content as string;
+      expect(content).toContain("[TIP:");
+      expect(content).toContain("loadImageAsBase64");
+      expect(content).toContain("/tmp/page-2026-01-01T12-00-00.png");
+    });
+
+    it("should detect multiple image paths and hint about all of them", () => {
+      const toolResponse =
+        "Before: ./before.png\nAfter: ./after.jpg\nDiff: ./diff.png";
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: toolResponse,
+          tool_call_id: "call_multi",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      const content = modifiedMessages[0].content as string;
+      expect(content).toContain("[TIP:");
+      expect(content).toContain("./before.png");
+      expect(content).toContain("./after.jpg");
+      expect(content).toContain("./diff.png");
+    });
+
+    it("should not add hint to messages that are actual base64 image data (already converted)", () => {
+      const validPngBase64 =
+        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: validPngBase64,
+          tool_call_id: "call_base64",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      // Should be converted to image array, not get a text hint
+      const content = modifiedMessages[0].content;
+      if (Array.isArray(content)) {
+        // Good - was converted to image content, no hint needed
+        expect(content[0]).toHaveProperty("type", "image_url");
+      } else {
+        // If kept as string, the hint should NOT be about a file path
+        // because base64 data URLs don't contain file paths
+        expect(content as string).not.toMatch(/loadImageAsBase64\("data:/);
+      }
+    });
+  });
+});