@tyvm/knowhow 0.0.125 → 0.0.127

package/PROCESSING.md

@@ -117,6 +117,27 @@ Video files are processed by:
 3. Analyzing visual content of each keyframe
 4. Combining transcript and visual analysis with timestamps
 
+### Keyframe Extraction Interval
+
+The `interval` parameter controls how often (in seconds) a frame is sampled from the video during keyframe extraction. It is accepted by both `streamKeyFrameExtraction()` and `extractKeyframes()` with a default of **10 seconds**.
+
+| Value | Effect |
+|-------|--------|
+| `5`   | Sample a frame every 5 seconds — more frames, finer detail |
+| `10`  | Default — one frame every 10 seconds, balanced coverage |
+| `30`  | One frame every 30 seconds — fewer frames, lower cost |
+
+**Example usage (programmatic):**
+```typescript
+// Extract a keyframe every 5 seconds
+await mediaProcessorService.extractKeyframes(filePath, outputPath, true, 5);
+
+// Stream keyframes every 30 seconds
+for await (const frame of mediaProcessorService.streamKeyFrameExtraction(filePath, videoJsonPath, true, 30)) {
+  console.log(frame);
+}
+```
+
 The actual output format from `convertVideoToText()` includes:
 ```
 Chunk: (1/10):

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.125",
+  "version": "0.0.127",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {

package/src/agents/base/base.ts

@@ -1001,8 +1001,9 @@ export abstract class BaseAgent implements IAgent {
         this.logStatus();
 
         const continuation = `<Workflow>
-        workflow continues until you call one of ${JSON.stringify(this.requiredToolNames)}.\n
-        ${statusMessage}
+        Task terminates after you call on of these tools: ${JSON.stringify(this.requiredToolNames)}.\n
+        User likely only sees output from the required tool call.
+        <TaskStatus>${statusMessage}</TaskStatus>
         </Workflow>`;
 
         messages.push({
@@ -1060,7 +1061,7 @@ export abstract class BaseAgent implements IAgent {
   }
 
   getStatusMessage() {
-    const baseMessage = `$${this.getTotalCostUsd().toPrecision(
+    const baseMessage = `Task spend: $${this.getTotalCostUsd().toPrecision(
       3
     )}\nElapsed: ${Math.floor(this.runTime() / 1000)}s\n`;
 

package/src/processors/TokenCompressor.ts

@@ -277,14 +277,60 @@ export class TokenCompressor implements JsonCompressorStorage {
     }
   }
 
+  /**
+   * Tool names whose responses should never be compressed.
+   * These are compression/retrieval tools themselves - compressing their output
+   * would prevent the agent from seeing the uncompressed data it just retrieved.
+   */
+  static readonly COMPRESSION_TOOL_NAMES: ReadonlySet<string> = new Set([
+    "expandTokens",
+    "grepToolResponse",
+    "jqToolResponse",
+    "tailToolResponse",
+    "listStoredToolResponses",
+  ]);
+
   createProcessor(
     filterFn?: (msg: Message) => boolean
   ): MessageProcessorFunction {
     return async (originalMessages: Message[], modifiedMessages: Message[]) => {
-      for (const message of modifiedMessages) {
+      // Find the index of the last assistant message so we only compress
+      // the most recent batch of tool responses (after the last agent output).
+      // This prevents re-compressing already-cached historical messages.
+      let lastAssistantIndex = -1;
+      for (let i = modifiedMessages.length - 1; i >= 0; i--) {
+        if (modifiedMessages[i].role === "assistant") {
+          lastAssistantIndex = i;
+          break;
+        }
+      }
+
+      for (let i = 0; i < modifiedMessages.length; i++) {
+        const message = modifiedMessages[i];
+
         if (filterFn && !filterFn(message)) {
           continue;
         }
+
+        // Only compress tool messages that are part of the last batch
+        // (i.e., they appear after the last assistant message).
+        // Messages before the last assistant message are historical and must
+        // not be modified to preserve prompt caching.
+        if (message.role === "tool" && i <= lastAssistantIndex) {
+          continue;
+        }
+
+        // Skip responses from compression-related tools — the agent called
+        // these specifically to see uncompressed data, so we must not
+        // re-compress their output.
+        if (
+          message.role === "tool" &&
+          message.name &&
+          TokenCompressor.COMPRESSION_TOOL_NAMES.has(message.name)
+        ) {
+          continue;
+        }
+
         await this.compressMessage(message);
       }
     };

package/src/services/MediaProcessorService.ts

@@ -26,6 +26,26 @@ export interface KeyframeInfo {
   usd_cost?: number;
 }
 
+/**
+ * Options for processing a full video (transcription + keyframes).
+ * Prefer passing this object over positional args so new options can be
+ * added without changing call sites.
+ */
+export interface ProcessVideoOptions {
+  /** Reuse a previously cached transcript/keyframes if present. Default true. */
+  reusePreviousTranscript?: boolean;
+  /** Audio transcription chunk length in seconds. Default 30. */
+  chunkTime?: number;
+  /** Progress callback (0..1) fired during ffmpeg audio chunking. */
+  onChunkingProgress?: (fraction: number) => void;
+  /**
+   * Seconds between sampled keyframes. When omitted, an adaptive interval is
+   * computed from the video duration (clamped 0.5s..30s), so short videos get
+   * a much denser set of frames (down to 500ms).
+   */
+  keyframeInterval?: number;
+}
+
 /**
  * MediaProcessorService handles audio/video processing using:
  * - ffmpeg (system tool) for chunking audio/video
@@ -272,13 +292,25 @@ export class MediaProcessorService {
 
   /**
    * Extract keyframes from a video file using ffmpeg, then describe each with vision AI.
+   *
+   * @param filePath - Path to the input video file.
+   * @param videoJsonPath - Path where the resulting keyframe JSON data will be saved/cached.
+   * @param reusePreviousKeyframes - Whether to reuse previously extracted keyframes (default: true).
+   * @param interval - How often (in seconds) to sample a frame from the video (default: 10).
+   *                   Lower values produce more frames; higher values produce fewer.
    */
   public async *streamKeyFrameExtraction(
     filePath: string,
     videoJsonPath: string,
     reusePreviousKeyframes = true,
-    interval = 10
+    interval?: number
   ): AsyncGenerator<KeyframeInfo> {
+    if (interval === undefined) {
+      const duration = await this.getMediaDuration(filePath);
+      const rawInterval = (duration / 3600) * 30;
+      interval = Math.min(30, Math.max(0.5, rawInterval));
+    }
+
     if (reusePreviousKeyframes && fs.existsSync(videoJsonPath)) {
       const contents = await readFile(videoJsonPath);
       const data = JSON.parse(contents.toString()) as KeyframeInfo[];
@@ -335,11 +367,21 @@ export class MediaProcessorService {
     );
   }
 
+  /**
+   * Extract keyframes from a video file and return them as an array.
+   *
+   * @param filePath - Path to the input video file.
+   * @param outputPath - Path where the resulting keyframe JSON data will be saved/cached.
+   * @param reusePreviousKeyframes - Whether to reuse previously extracted keyframes (default: true).
+   * @param interval - How often (in seconds) to sample a frame from the video (default: 10).
+   *                   Lower values produce more frames; higher values produce fewer.
+   */
+
   public async extractKeyframes(
     filePath: string,
     outputPath: string,
     reusePreviousKeyframes = true,
-    interval = 10
+    interval?: number
   ): Promise<KeyframeInfo[]> {
     const keyframes: KeyframeInfo[] = [];
     for await (const keyframe of this.streamKeyFrameExtraction(
@@ -354,6 +396,110 @@ export class MediaProcessorService {
     return keyframes;
   }
 
+  /**
+   * Extract (and optionally describe) keyframes for a specific time slice of a
+   * video, e.g. "just the first minute". This is useful for focusing a denser
+   * frame sample on a short region without reprocessing the whole video.
+   *
+   * @param filePath   Path to the input video file.
+   * @param startSec   Start of the slice in seconds (inclusive).
+   * @param endSec     End of the slice in seconds (exclusive).
+   * @param options.interval  Seconds between sampled frames within the slice.
+   *                          Defaults to 1s. Clamped to a minimum of 0.5s.
+   * @param options.describe  Whether to run the vision model on each frame.
+   *                          Defaults to true. Set false to get frames only.
+   * @param options.outputDir Directory to write extracted frames into.
+   *                          Defaults to <dir>/<name>/slices/<start>-<end>.
+   */
+  public async *streamKeyFrameExtractionForRange(
+    filePath: string,
+    startSec: number,
+    endSec: number,
+    options: {
+      interval?: number;
+      describe?: boolean;
+      outputDir?: string;
+    } = {}
+  ): AsyncGenerator<KeyframeInfo> {
+    const interval = Math.max(0.5, options.interval ?? 1);
+    const describe = options.describe ?? true;
+    const parsed = path.parse(filePath);
+    const outputDir =
+      options.outputDir ??
+      path.join(parsed.dir, parsed.name, "slices", `${startSec}-${endSec}`);
+    await fs.promises.mkdir(outputDir, { recursive: true });
+
+    const duration = Math.max(0, endSec - startSec);
+    // -ss before -i seeks quickly; -t limits to the slice length.
+    const command =
+      `ffmpeg -y -ss ${startSec} -t ${duration} -i "${filePath}" ` +
+      `-vf "fps=1/${interval},scale=640:-1" "${outputDir}/frame%04d.jpg"`;
+    await execAsync(command);
+
+    const frames = (await fs.promises.readdir(outputDir))
+      .filter((f) => f.endsWith(".jpg"))
+      .sort();
+
+    for (const frame of frames) {
+      const framePath = path.join(outputDir, frame);
+      const frameNumber = parseInt(frame.match(/\d+/)?.[0] ?? "0", 10);
+      // ffmpeg names frames starting at 1; map back to absolute video time.
+      const timestamp = startSec + (frameNumber - 1) * interval;
+
+      if (!describe) {
+        yield { path: framePath, description: "", timestamp, usd_cost: 0 };
+        continue;
+      }
+
+      const description = await this.describeKeyframe(framePath);
+      yield {
+        path: framePath,
+        description: description.choices[0].message.content,
+        timestamp,
+        usd_cost: description.usd_cost,
+      };
+    }
+  }
+
+  /**
+   * Extract an audio slice [startSec, endSec) from a media file into an mp3.
+   * Returns the path to the produced mp3, suitable for transcription or download.
+   */
+  public async extractAudioSlice(
+    filePath: string,
+    startSec: number,
+    endSec: number,
+    outputPath?: string
+  ): Promise<string> {
+    const parsed = path.parse(filePath);
+    const outPath =
+      outputPath ??
+      path.join(
+        parsed.dir,
+        parsed.name,
+        "slices",
+        `audio-${startSec}-${endSec}.mp3`
+      );
+    await fs.promises.mkdir(path.dirname(outPath), { recursive: true });
+
+    const duration = Math.max(0, endSec - startSec);
+    const command =
+      `ffmpeg -y -ss ${startSec} -t ${duration} -i "${filePath}" ` +
+      `-vn -acodec libmp3lame -ac 1 -b:a 32k "${outPath}"`;
+    await execAsync(command);
+    return outPath;
+  }
+
+  /**
+   * Get the duration of a media file in seconds using ffprobe.
+   */
+  public async getMediaDuration(filePath: string): Promise<number> {
+    const output = await execAsync(
+      `ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "${filePath}"`
+    );
+    return parseFloat(output.trim());
+  }
+
   private async describeKeyframe(keyframePath: string) {
     const question =
       "Describe this image in detail, focusing on the main elements and actions visible.";
@@ -379,40 +525,98 @@ export class MediaProcessorService {
 
   async *streamProcessVideo(
     filePath: string,
-    reusePreviousTranscript = true,
+    reusePreviousTranscriptOrOptions: boolean | ProcessVideoOptions = true,
     chunkTime = 30,
-    onChunkingProgress?: (fraction: number) => void
+    onChunkingProgress?: (fraction: number) => void,
+    keyframeInterval?: number
   ) {
+    // Support both the legacy positional signature and a new options object
+    // passed as the second argument. Prefer the options object going forward.
+    const opts: Required<Pick<ProcessVideoOptions, "reusePreviousTranscript" | "chunkTime">> &
+      ProcessVideoOptions =
+      typeof reusePreviousTranscriptOrOptions === "object"
+        ? {
+            reusePreviousTranscript:
+              reusePreviousTranscriptOrOptions.reusePreviousTranscript ?? true,
+            chunkTime: reusePreviousTranscriptOrOptions.chunkTime ?? 30,
+            onChunkingProgress:
+              reusePreviousTranscriptOrOptions.onChunkingProgress,
+            keyframeInterval:
+              reusePreviousTranscriptOrOptions.keyframeInterval,
+          }
+        : {
+            reusePreviousTranscript: reusePreviousTranscriptOrOptions,
+            chunkTime,
+            onChunkingProgress,
+            keyframeInterval,
+          };
+
     const parsed = path.parse(filePath);
     const videoJson = `${parsed.dir}/${parsed.name}/video.json`;
 
     console.log("Processing audio...");
     const transcriptions = this.streamProcessAudio(
       filePath,
-      reusePreviousTranscript,
-      chunkTime,
-      onChunkingProgress
+      opts.reusePreviousTranscript,
+      opts.chunkTime,
+      opts.onChunkingProgress
     );
 
     console.log("Extracting keyframes...");
+    // When keyframeInterval is undefined, streamKeyFrameExtraction computes an
+    // adaptive interval based on video duration, clamped between 0.5s and 30s.
+    // This gives short videos a much denser set of frames (down to 500ms) while
+    // keeping long videos at a reasonable frame count. Previously this passed
+    // chunkTime (30s), which meant every video only got 1 frame per 30s.
     const videoAnalysis = this.streamKeyFrameExtraction(
       filePath,
       videoJson,
-      reusePreviousTranscript,
-      chunkTime
+      opts.reusePreviousTranscript,
+      opts.keyframeInterval
     );
 
+    // Frames and transcript chunks can arrive at different cadences: with an
+    // adaptive keyframe interval a short video may have many more frames than
+    // 30s transcript chunks. Rather than a 1:1 positional zip (which would
+    // strand most frames with "[missing transcript]" and drop transcripts),
+    // we map each frame to the transcript chunk that covers its timestamp.
+    // Transcript chunk N covers [N*chunkTime, (N+1)*chunkTime).
+    const transcriptBuffer: TranscriptChunk[] = [];
+    const missingTranscript: TranscriptChunk = {
+      chunkPath: "",
+      text: "[missing transcript]",
+      usd_cost: 0,
+    };
+
     for await (const frame of videoAnalysis) {
-      const transcription = (await transcriptions.next())
-        ?.value as TranscriptChunk;
+      const chunkIndex = Math.floor((frame.timestamp || 0) / opts.chunkTime);
+
+      // Pull transcript chunks until we have one that covers this frame's window.
+      while (transcriptBuffer.length <= chunkIndex) {
+        const next = await transcriptions.next();
+        if (next.done) break;
+        transcriptBuffer.push(next.value as TranscriptChunk);
+      }
+
+      const transcription = transcriptBuffer[chunkIndex] || missingTranscript;
+      yield { frame, transcription };
+    }
+
+    // Drain any remaining transcript chunks (e.g. audio longer than the last
+    // frame's window) so their content/cost isn't silently dropped.
+    let remaining = await transcriptions.next();
+    while (!remaining.done) {
       yield {
-        frame,
-        transcription: transcription || {
-          chunkPath: "",
-          text: "[missing transcript]",
+        frame: {
+          path: "",
+          description: "",
+          timestamp: transcriptBuffer.length * opts.chunkTime,
           usd_cost: 0,
         },
+        transcription: remaining.value as TranscriptChunk,
       };
+      transcriptBuffer.push(remaining.value as TranscriptChunk);
+      remaining = await transcriptions.next();
     }
   }
 

package/src/services/conversion/ConversionService.ts

@@ -151,21 +151,10 @@ export class ConversionService {
       },
     });
 
-    // ── text passthrough (known extensions) ────────────────────────────────
-    this.register({
-      name: "text-passthrough",
-      inputExts: ["txt", "md", "json", "yaml", "yml", "csv", "xml", "html", "htm", "js", "ts", "py", "rb", "sh", "text"],
-      outputType: "text",
-      async convert(input, _ctx): Promise<ConvertResult> {
-        const text = fs.readFileSync(input.filePath, "utf8");
-        return { outputType: "text", text };
-      },
-    });
-
-    // ── text passthrough fallback ───────────────────────────────────────────
+    // ── text passthrough (catch-all: any file is assumed to be readable as text) ─
     this.register({
       name: "text-passthrough-fallback",
-      inputModality: "text",
+      catchAll: true,
       outputType: "text",
       async convert(input, _ctx): Promise<ConvertResult> {
         const text = fs.readFileSync(input.filePath, "utf8");
@@ -467,16 +456,7 @@ export class ConversionService {
     const queue: State[] = [{ node: inputNode, path: [] }];
     const visited = new Set<string>();
     visited.add(inputNode);
-
-    const knownModalities: Modality[] = ["text", "html", "image", "audio", "video"];
-    const isKnownModality = knownModalities.includes(inputNode as Modality);
-    const hasDirectMatch = this.converters.some(
-      (c) => c.inputExts?.includes(inputNode) || c.inputModality === inputNode
-    );
-    if (!isKnownModality && !hasDirectMatch && inputNode !== "text") {
-      queue.push({ node: "text", path: [] });
-      visited.add("text");
-    }
+    let catchAllPath: Converter[] | null = null;
 
     while (queue.length > 0) {
       const { node, path } = queue.shift()!;
@@ -484,13 +464,21 @@ export class ConversionService {
       const candidates = this.converters.filter((c) => {
         if (c.inputExts && c.inputExts.includes(node)) return true;
         if (c.inputModality && c.inputModality === node) return true;
+        // catch-all converters match any node (only at the start node, not intermediate)
+        if (c.catchAll && node === inputNode) return true;
         return false;
       });
 
       for (const converter of candidates) {
         const newPath = [...path, converter];
         if (converter.outputType === targetType) {
-          return [newPath];
+          if (converter.catchAll) {
+            // Defer catch-all paths — only use if no specific path is found
+            if (!catchAllPath) catchAllPath = newPath;
+          } else {
+            return [newPath];
+          }
+          continue;
         }
         if (!visited.has(converter.outputType)) {
           visited.add(converter.outputType);
@@ -499,7 +487,8 @@ export class ConversionService {
       }
     }
 
-    return null;
+    // Fall back to catch-all path if no specific path was found
+    return catchAllPath ? [catchAllPath] : null;
   }
 
   /**
@@ -742,10 +731,15 @@ export class ConversionService {
       return inputMatch && c.outputType === outputType;
     });
 
+    // Append catch-all converters as last-resort fallbacks (only if output type matches)
+    const catchAlls = this.converters.filter(
+      (c) => c.catchAll && c.outputType === outputType && !matching.includes(c)
+    );
+
     const preferred: Converter[] = [];
     const rest: Converter[] = [];
 
-    for (const c of matching) {
+    for (const c of [...matching, ...catchAlls]) {
       if (preferredNames.includes(c.name)) {
         preferred.push(c);
       } else {

package/src/services/conversion/types.ts

@@ -41,6 +41,14 @@ export interface Converter {
   name: string;                   // unique
   inputExts?: string[];           // e.g. ["pdf"]
   inputModality?: Modality;       // OR an input modality (e.g. "image")
+  /**
+   * If true, this converter is used as a last-resort fallback for any input
+   * type that has no other converter registered. Catch-all converters are
+   * appended after all specific matches in buildAlternatives, and are also
+   * added as BFS nodes so they can be discovered even for unknown extensions.
+   * Use sparingly — only for truly generic converters like text-passthrough.
+   */
+  catchAll?: boolean;
   outputType: Modality;
   /**
    * If true, the ConversionService will cache the result of this converter to

package/src/services/modules/index.ts

@@ -1,5 +1,6 @@
 import * as path from "path";
 import * as os from "os";
+import * as Module from "module";
 
 import { getConfig, getGlobalConfig } from "../../config";
 import { KnowhowModule, ModuleContext } from "./types";
@@ -25,6 +26,16 @@ export class ModulesService {
       context = { ...(await this.getDefaultContext()) };
     }
 
+    // Prepend the global knowhow install's own node_modules to Node's global module
+    // search paths so that locally-installed modules (loaded from .knowhow/node_modules)
+    // that depend on @tyvm/knowhow resolve to the current global version rather than
+    // a stale bundled copy.
+    const globalKnowhowNodeModules = path.join(__dirname, "../../../../node_modules");
+    const globalPaths = (Module as any).globalPaths as string[];
+    if (globalPaths && !globalPaths.includes(globalKnowhowNodeModules)) {
+      globalPaths.unshift(globalKnowhowNodeModules);
+    }
+
     const allModulePaths = config.modules;
 
     // Search paths: local .knowhow/node_modules first (where `knowhow modules install`