@tyvm/knowhow 0.0.108 → 0.0.109-dev.38b1faa

package/src/clients/anthropic.ts

@@ -40,11 +40,11 @@ export class GenericAnthropicClient implements GenericClient {
     });
   }
 
-  handleToolCaching(tools: Anthropic.Tool[]) {
+  handleToolCaching(tools: Anthropic.Tool[], longTtl = false) {
     const lastTool = tools[tools.length - 1];
 
     if (lastTool) {
-      lastTool.cache_control = { type: "ephemeral" };
+      lastTool.cache_control = longTtl ? { type: "ephemeral", ttl: "1h" } as any : { type: "ephemeral" };
     }
   }
 
@@ -94,7 +94,7 @@ export class GenericAnthropicClient implements GenericClient {
     return cleaned;
   }
 
-  transformTools(tools?: Tool[]): Anthropic.Tool[] {
+  transformTools(tools?: Tool[], longTtl = false): Anthropic.Tool[] {
     if (!tools) {
       return [];
     }
@@ -104,7 +104,7 @@ export class GenericAnthropicClient implements GenericClient {
       input_schema: this.cleanSchemaForAnthropic(tool.function.parameters) as any,
     }));
 
-    this.handleToolCaching(transformed);
+    this.handleToolCaching(transformed, longTtl);
 
     return transformed;
   }
@@ -153,16 +153,16 @@ export class GenericAnthropicClient implements GenericClient {
     return messages;
   }
 
-  cacheLastContent(message: MessageParam) {
+  cacheLastContent(message: MessageParam, longTtl = false) {
     if (Array.isArray(message.content)) {
       const lastMessage = message.content[message.content.length - 1];
       if (
         lastMessage.type !== "thinking" &&
         lastMessage.type !== "redacted_thinking"
       ) {
-        lastMessage.cache_control = {
-          type: "ephemeral",
-        };
+        lastMessage.cache_control = longTtl
+          ? ({ type: "ephemeral", ttl: "1h" } as any)
+          : { type: "ephemeral" };
       }
     }
   }
@@ -179,7 +179,7 @@ export class GenericAnthropicClient implements GenericClient {
     }
   }
 
-  handleMessageCaching(groupedMessages: MessageParam[]) {
+  handleMessageCaching(groupedMessages: MessageParam[], longTtl = false) {
     this.handleClearingCache(groupedMessages);
 
     // find the last two messages and mark them as ephemeral
@@ -189,7 +189,7 @@ export class GenericAnthropicClient implements GenericClient {
 
     for (const m of lastTwoUserMessages) {
       if (Array.isArray(m.content)) {
-        this.cacheLastContent(m);
+        this.cacheLastContent(m, longTtl);
       }
     }
   }
@@ -203,7 +203,7 @@ export class GenericAnthropicClient implements GenericClient {
     }
   }
 
-  transformMessages(messages: Message[]): MessageParam[] {
+  transformMessages(messages: Message[], longTtl = false): MessageParam[] {
     const toolCalls = messages.flatMap((msg) => msg.tool_calls || []);
     const claudeMessages: MessageParam[] = messages
       .filter((msg) => msg.role !== "system")
@@ -302,7 +302,7 @@ export class GenericAnthropicClient implements GenericClient {
 
     const groupedMessages = this.combineMessages(claudeMessages);
 
-    this.handleMessageCaching(groupedMessages);
+    this.handleMessageCaching(groupedMessages, longTtl);
 
     return groupedMessages;
   }
@@ -349,14 +349,15 @@ export class GenericAnthropicClient implements GenericClient {
   async createChatCompletion(
     options: CompletionOptions
   ): Promise<CompletionResponse> {
+    const longTtl = !!options.long_ttl_cache;
     const systemMessage = options.messages
       .filter((msg) => msg.role === "system")
       .map((msg) => msg.content || "")
       .join("\n");
 
-    const claudeMessages = this.transformMessages(options.messages);
+    const claudeMessages = this.transformMessages(options.messages, longTtl);
 
-    const tools = this.transformTools(options.tools);
+    const tools = this.transformTools(options.tools, longTtl);
     try {
       const response = await this.client.messages.create({
         model: options.model,
@@ -365,7 +366,7 @@ export class GenericAnthropicClient implements GenericClient {
           ? [
               {
                 text: systemMessage,
-                cache_control: { type: "ephemeral" },
+                cache_control: longTtl ? ({ type: "ephemeral", ttl: "1h" } as any) : { type: "ephemeral" },
                 type: "text",
               },
             ]
@@ -375,7 +376,7 @@ export class GenericAnthropicClient implements GenericClient {
           tool_choice: { type: "auto" },
           tools,
         }),
-      });
+      }, { signal: options.signal });
 
       if (!response.content || !response.content.length) {
         console.log("no content in Anthropic response", response);
@@ -424,7 +425,7 @@ export class GenericAnthropicClient implements GenericClient {
         usd_cost: this.calculateCost(options.model, response.usage),
       };
     } catch (err) {
-      if ("headers" in err && err.headers["x-should-retry"] === "true") {
+      if ("headers" in err && err.headers?.["x-should-retry"] === "true") {
         console.warn("Retrying failed request", err);
         await wait(2500);
         return this.createChatCompletion(options);

package/src/clients/index.ts

@@ -33,6 +33,7 @@ import { ContextLimits } from "./contextLimits";
 import { OpenAiTextPricing } from "./pricing/openai";
 import { AnthropicTextPricing } from "./pricing/anthropic";
 import { GeminiPricing } from "./pricing/google";
+import { withRetry } from "./withRetry";
 import {
   XaiTextPricing,
   XaiImagePricing,
@@ -577,11 +578,12 @@ export class AIClient {
    * @param modelQuery - the model name to search for (can be partial/normalized)
    * @param provider   - optional provider to restrict search to
    */
-  findModelFuzzy(modelQuery: string, provider?: string): { provider: string; model: string } | undefined {
+  findModelFuzzy(
+    modelQuery: string,
+    provider?: string
+  ): { provider: string; model: string } | undefined {
     const queryNorm = AIClient.normalizeModelId(modelQuery);
-    const providers = provider
-      ? [provider]
-      : Object.keys(this.clientModels);
+    const providers = provider ? [provider] : Object.keys(this.clientModels);
 
     for (const p of providers) {
       const models = (this.clientModels[p] as string[]) ?? [];
@@ -664,7 +666,10 @@ export class AIClient {
         } model registered. Try using ${JSON.stringify(this.listAllModels())}`
       );
     }
-    return client.createChatCompletion({ ...options, model });
+    return withRetry(
+      (signal) => client.createChatCompletion({ ...options, model, signal }),
+      options
+    );
   }
 
   async createEmbedding(
@@ -679,7 +684,10 @@ export class AIClient {
         } model registered. Try using ${JSON.stringify(this.listAllModels())}`
       );
     }
-    return client.createEmbedding({ ...options, model });
+    return withRetry(
+      (signal) => client.createEmbedding({ ...options, model, signal }),
+      options
+    );
   }
 
   async createAudioTranscription(
@@ -692,7 +700,10 @@ export class AIClient {
         `Provider ${provider} does not support audio transcription.`
       );
     }
-    return client.createAudioTranscription(options);
+    return withRetry(
+      (signal) => client.createAudioTranscription({ ...options, signal }),
+      options
+    );
   }
 
   async createAudioGeneration(
@@ -710,7 +721,10 @@ export class AIClient {
         `Model ${options.model} not registered for provider ${provider}.`
       );
     }
-    return client.createAudioGeneration({ ...options, model });
+    return withRetry(
+      (signal) => client.createAudioGeneration({ ...options, model, signal }),
+      options
+    );
   }
 
   async createImageGeneration(
@@ -728,7 +742,10 @@ export class AIClient {
         `Model ${options.model} not registered for provider ${provider}.`
       );
     }
-    return client.createImageGeneration({ ...options, model });
+    return withRetry(
+      (signal) => client.createImageGeneration({ ...options, model, signal }),
+      options
+    );
   }
 
   async createVideoGeneration(
@@ -746,7 +763,10 @@ export class AIClient {
         `Model ${options.model} not registered for provider ${provider}.`
       );
     }
-    return client.createVideoGeneration({ ...options, model });
+    return withRetry(
+      (signal) => client.createVideoGeneration({ ...options, model, signal }),
+      options
+    );
   }
 
   async getVideoStatus(
@@ -835,7 +855,7 @@ export class AIClient {
         const splitModel = m.id.split("/");
 
         if (splitModel.length < 2) {
-          console.error(`Cannot parse model format: ${m.id}`);
+          console.warn(`Cannot parse model format: ${m.id}`);
         }
 
         const provider = splitModel.length > 1 ? splitModel[0] : "";

package/src/clients/openai.ts

@@ -63,6 +63,10 @@ export class GenericOpenAiClient implements GenericClient {
     });
   }
 
+  /**
+   * Execute a function with timeout, retries, and exponential backoff.
+   * Retriable errors: 5xx, timeout, ECONNRESET, ETIMEDOUT, rate limits (429).
+   */
   reasoningEffort(
     messages: CompletionOptions["messages"]
   ): "low" | "medium" | "high" {
@@ -155,12 +159,11 @@ export class GenericOpenAiClient implements GenericClient {
         max_completion_tokens: Math.max(options.max_tokens ?? 0, 16_000),
         reasoning_effort: this.resolveReasoningEffort(options),
       }),
-
       ...(options.tools && {
         tools: options.tools,
         tool_choice: "auto",
       }),
-    });
+    }, { signal: options.signal });
 
     const usdCost = this.calculateCost(options.model, response.usage);
 
@@ -453,7 +456,7 @@ export class GenericOpenAiClient implements GenericClient {
       prompt: options.prompt,
       response_format: options.response_format || "verbose_json",
       temperature: options.temperature,
-    });
+    }, { signal: options.signal });
 
     // Calculate cost: $0.006 per minute for Whisper
     const duration = typeof response === "object" && "duration" in response && typeof response.duration === "number"
@@ -489,7 +492,7 @@ export class GenericOpenAiClient implements GenericClient {
       voice: options.voice as any,
       response_format: options.response_format || "mp3",
       speed: options.speed,
-    });
+    }, { signal: options.signal });
 
     const buffer = Buffer.from(await response.arrayBuffer());
 
@@ -518,7 +521,7 @@ export class GenericOpenAiClient implements GenericClient {
       style: options.style,
       response_format: options.response_format,
       user: options.user,
-    });
+    }, { signal: options.signal });
 
     // Cost calculation varies by model and settings
     // DALL-E 3: $0.040-$0.120 per image depending on quality/size

package/src/clients/types.ts

@@ -1,4 +1,10 @@
-export type ModelModality = "completion" | "embedding" | "image" | "audio" | "video" | "transcription";
+export type ModelModality =
+  | "completion"
+  | "embedding"
+  | "image"
+  | "audio"
+  | "video"
+  | "transcription";
 
 export type MessageContent =
   | { type: "text"; text: string }
@@ -8,7 +14,7 @@ export type MessageContent =
 
 export interface Message {
   role: "system" | "user" | "assistant" | "tool";
-  content?: string | MessageContent[];
+  content?: string | MessageContent[] | null;
 
   name?: string;
   tool_call_id?: string;
@@ -16,7 +22,7 @@ export interface Message {
 }
 
 export interface OutputMessage extends Message {
-  content: string;
+  content?: string | null;
 }
 
 export interface ToolProp {
@@ -51,7 +57,30 @@ export interface ToolCall {
   };
 }
 
-export interface CompletionOptions {
+export interface RetryOptions {
+  /**
+   * Request timeout in milliseconds per attempt. If the request does not complete
+   * within this time it is aborted and retried according to maxRetries.
+   */
+  timeout?: number;
+  /**
+   * Maximum number of retry attempts for retriable errors (5xx, timeout, ECONNRESET, 429).
+   * Default: 2. Set to 0 to disable retries.
+   */
+  maxRetries?: number;
+  /**
+   * Base backoff delay in milliseconds for exponential retry backoff.
+   * Default: 1000ms. Each retry waits backoffMs * 2^attempt ms.
+   */
+  backoffMs?: number;
+  /**
+   * Optional external AbortSignal. When the signal is aborted the current
+   * attempt is cancelled immediately and no further retries are made.
+   */
+  signal?: AbortSignal;
+}
+
+export interface CompletionOptions extends RetryOptions {
   model: string;
   messages: Message[];
   tools?: Tool[];
@@ -61,6 +90,13 @@ export interface CompletionOptions {
    *  Maps to: OpenAI reasoning_effort, xAI reasoning.effort, Gemini thinkingLevel/thinkingBudget, Anthropic thinking budget.
    *  "low" = minimal thinking, "medium" = balanced, "high" = maximum reasoning */
   reasoning_effort?: "low" | "medium" | "high";
+  /**
+   * When true, hints to the client that this task is long-running and it should
+   * use a long-TTL cache where available.
+   * - Anthropic: enables the `extended-cache-ttl-2025-02-19` beta and sets
+   *   `cache_control.ttl` to 3600 (1 hour) instead of the default 5-minute ephemeral cache.
+   */
+  long_ttl_cache?: boolean;
 }
 
 /**
@@ -100,7 +136,7 @@ export interface CompletionResponse {
   usd_cost?: number;
 }
 
-export interface EmbeddingOptions {
+export interface EmbeddingOptions extends RetryOptions {
   input: string;
   model?: string;
 }
@@ -119,7 +155,7 @@ export interface EmbeddingResponse {
   usd_cost?: number;
 }
 
-export interface AudioTranscriptionOptions {
+export interface AudioTranscriptionOptions extends RetryOptions {
   file: Blob | File | any; // Support for Node.js ReadStream or web File/Blob
   model?: string;
   language?: string;
@@ -149,7 +185,7 @@ export interface AudioTranscriptionResponse {
   usd_cost?: number;
 }
 
-export interface AudioGenerationOptions {
+export interface AudioGenerationOptions extends RetryOptions {
   model: string;
   input: string;
   voice: string; // e.g. "alloy", "echo", "fable", "onyx", "nova", "shimmer" for OpenAI; "Kore", "Puck" etc. for Gemini
@@ -163,7 +199,7 @@ export interface AudioGenerationResponse {
   usd_cost?: number;
 }
 
-export interface ImageGenerationOptions {
+export interface ImageGenerationOptions extends RetryOptions {
   model: string;
   prompt: string;
   n?: number;
@@ -184,7 +220,7 @@ export interface ImageGenerationResponse {
   usd_cost?: number;
 }
 
-export interface VideoGenerationOptions {
+export interface VideoGenerationOptions extends RetryOptions {
   model: string;
   prompt: string;
   duration?: number; // seconds
@@ -294,7 +330,9 @@ export interface GenericClient {
    * When modality is provided, return only models for that modality (static list).
    * When omitted, return ALL models (backward compat — may do a live API call).
    */
-  getModels(modality?: ModelModality): Promise<{ id: string; modality?: ModelModality[] }[]>;
+  getModels(
+    modality?: ModelModality
+  ): Promise<{ id: string; modality?: ModelModality[] }[]>;
   /**
    * Returns the context window limit and compression threshold for a given model,
    * or undefined if the model is not known to this client.

package/src/clients/withRetry.ts

@@ -0,0 +1,89 @@
+/**
+ * Shared retry/timeout helper for all AI clients.
+ *
+ * Executes `fn` with exponential backoff for retriable errors:
+ * - Rate limits (429)
+ * - Timeouts (AbortError, ETIMEDOUT, ECONNRESET)
+ * - Server errors (5xx)
+ *
+ * @param fn               Function to execute. Receives a combined AbortSignal
+ *                         that fires on per-attempt timeout OR external signal abort.
+ * @param opts             Any object with optional RetryOptions fields (timeout, maxRetries,
+ *                         backoffMs, signal). Extra fields are ignored — so you can pass the
+ *                         full options object from any AI method directly.
+ *                         - timeout: Per-attempt timeout in ms. No timeout if omitted.
+ *                         - maxRetries: Max retry attempts after first failure. Default: 2.
+ *                         - backoffMs: Base backoff delay in ms. Default: 1000.
+ *                         - signal: Optional external AbortSignal. When aborted, the current
+ *                         attempt is cancelled and no further retries are made.
+ */
+import type { RetryOptions } from "./types";
+
+export async function withRetry<T>(
+  fn: (signal?: AbortSignal) => Promise<T>,
+  opts: RetryOptions = {}
+): Promise<T> {
+  const maxRetries = opts.maxRetries ?? 2;
+  const backoffMs = opts.backoffMs ?? 1000;
+  const timeout = opts.timeout;
+  const externalSignal = opts.signal;
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    // If the external signal is already aborted, bail out immediately.
+    if (externalSignal?.aborted) {
+      throw externalSignal.reason ?? new DOMException("Aborted", "AbortError");
+    }
+
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    // Combine per-attempt timeout with the external signal into one controller.
+    const controller = timeout || externalSignal ? new AbortController() : undefined;
+
+    if (controller) {
+      if (timeout) {
+        timer = setTimeout(() => controller.abort(new DOMException("Request timed out", "TimeoutError")), timeout);
+      }
+      // Forward external signal abort into our combined controller.
+      if (externalSignal) {
+        const onExternalAbort = () => controller.abort(externalSignal.reason ?? new DOMException("Aborted", "AbortError"));
+        if (externalSignal.aborted) {
+          controller.abort(externalSignal.reason ?? new DOMException("Aborted", "AbortError"));
+        } else {
+          externalSignal.addEventListener("abort", onExternalAbort, { once: true });
+          // Clean up the listener after the attempt resolves/rejects.
+          controller.signal.addEventListener("abort", () =>
+            externalSignal.removeEventListener("abort", onExternalAbort), { once: true }
+          );
+        }
+      }
+    }
+
+    try {
+      const result = await fn(controller?.signal);
+      return result;
+    } catch (err: unknown) {
+      clearTimeout(timer);
+      // If the external signal was aborted, don't retry — propagate immediately.
+      if (externalSignal?.aborted) {
+        throw err;
+      }
+      const errStr = String(err);
+      const isRetriable =
+        errStr.includes('429') ||
+        errStr.includes('timeout') ||
+        errStr.includes('TimeoutError') ||
+        errStr.includes('ECONNRESET') ||
+        errStr.includes('ETIMEDOUT') ||
+        errStr.includes('AbortError') ||
+        /5\d\d/.test(errStr);
+      if (isRetriable && attempt < maxRetries) {
+        const delay = backoffMs * Math.pow(2, attempt);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+        continue;
+      }
+      throw err;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+  throw new Error('withRetry: exhausted retries');
+}