ai-retry 0.0.1 → 0.0.3

package/README.md

@@ -1,3 +1,5 @@
+<a href="https://www.npmjs.com/package/ai-retry" alt="ai-retry"><img src="https://img.shields.io/npm/dt/ai-retry?label=ai-retry"></a> <a href="https://github.com/zirkelc/ai-retry/actions/workflows/ci.yml" alt="CI"><img src="https://img.shields.io/github/actions/workflow/status/zirkelc/ai-retry/ci.yml?branch=main"></a>
+
 # ai-retry: Retry and fallback mechanisms for AI SDK
 
 Automatically handle API failures, content filtering and timeouts by switching between different AI models.
@@ -23,14 +25,14 @@ npm install ai-retry
 
 Create a retryable model by providing a base model and a list of retryables or fallback models.
 
-> [!WARNING]  
-> `ai-retry` currently only supports `generateText` and `generateObject` calls.
-> Streaming via `streamText` and `streamObject` is not supported yet.
+> [!NOTE]  
+> `ai-retry` currently supports `generateText`, `generateObject`, `streamText`, and `streamObject` calls.
+> Note that streaming retry has limitations: retries are only possible before content starts flowing or very early in the stream.
 
 ```typescript
 import { azure } from '@ai-sdk/azure';
 import { openai } from '@ai-sdk/openai';
-import { generateText } from 'ai';
+import { generateText, streamText } from 'ai';
 import { createRetryable } from 'ai-retry';
 import { contentFilterTriggered, requestTimeout } from 'ai-retry/retryables';
 
@@ -47,13 +49,25 @@ const result = await generateText({
   model: retryableModel,
   prompt: 'Hello world!',
 });
-```
 
+// Or with streaming
+const result = streamText({
+  model: retryableModel,
+  prompt: 'Write a story about a robot...',
+});
+
+for await (const chunk of result.textStream) {
+  console.log(chunk.text);
+}
+```
 
 #### Content Filter
 
 Automatically switch to a different model when content filtering blocks your request.
 
+> [!WARNING]  
+> This retryable currently does not work with streaming requests, because the content filter is only indicated in the final response.
+
 ```typescript
 import { contentFilterTriggered } from 'ai-retry/retryables';
 
@@ -118,6 +132,21 @@ const result = await generateText({
 });
 ```
 
+#### Service Overloaded
+
+Handle service overload errors (HTTP code 529) by switching to a provider.
+
+```typescript
+import { serviceOverloaded } from 'ai-retry/retryables';
+
+const retryableModel = createRetryable({
+  model: azure('gpt-4'),
+  retries: [
+    serviceOverloaded(openai('gpt-4')), // Switch to OpenAI if Azure is overloaded
+  ],
+});
+```
+
 #### Request Not Retryable
 
 Handle cases where the base model fails with a non-retryable error.
@@ -240,6 +269,7 @@ There are several built-in retryables:
 - [`contentFilterTriggered`](./src/retryables/content-filter-triggered.ts): Content filter was triggered based on the prompt or completion.
 - [`requestTimeout`](./src/retryables/request-timeout.ts): Request timeout occurred.
 - [`requestNotRetryable`](./src/retryables/request-not-retryable.ts): Request failed with a non-retryable error.
+- [`serviceOverloaded`](./src/retryables/service-overloaded.ts): Response with status code 529 (service overloaded).
 
 By default, each retryable will only attempt to retry once per model to avoid infinite loops. You can customize this behavior by returning a `maxAttempts` value from your retryable function.
 

package/dist/{create-retryable-model-CnrFowSg.js → create-retryable-model-C4nAHxnW.js}

@@ -9,6 +9,12 @@ const getModelKey = (model) => {
 	return `${model.provider}/${model.modelId}`;
 };
 
+//#endregion
+//#region src/utils.ts
+const isObject = (value) => typeof value === "object" && value !== null;
+const isString = (value) => typeof value === "string";
+const isGenerateResult = (result) => "content" in result;
+
 //#endregion
 //#region src/create-retryable-model.ts
 /**
@@ -77,19 +83,22 @@ var RetryableModel = class {
 			}
 		}
 	}
-	async doGenerate(options) {
+	/**
+	* Execute a function with retry logic for handling errors
+	*/
+	async executeWithRetry(fn, retryState) {
 		/**
 		* Always start with the original model
 		*/
-		this.currentModel = this.baseModel;
+		this.currentModel = retryState?.currentModel ?? this.baseModel;
 		/**
 		* Track number of attempts
 		*/
-		let totalAttempts = 0;
+		let totalAttempts = retryState?.totalAttempts ?? 0;
 		/**
 		* Track all attempts.
 		*/
-		const attempts = [];
+		const attempts = retryState?.attempts ?? [];
 		/**
 		* The previous attempt that triggered a retry, or undefined if this is the first attempt
 		*/
@@ -118,43 +127,48 @@ var RetryableModel = class {
 			}
 			totalAttempts++;
 			try {
-				const result = await this.currentModel.doGenerate(options);
+				const result = await fn();
 				/**
-				* Check if the result should trigger a retry
+				* Check if the result should trigger a retry (only for generate results, not streams)
 				*/
-				const resultAttempt = {
-					type: "result",
-					result,
-					model: this.currentModel
-				};
-				/**
-				* Add the current attempt to the list before checking for retries
-				*/
-				attempts.push(resultAttempt);
-				const resultContext = {
-					current: resultAttempt,
-					attempts,
-					totalAttempts
-				};
-				const nextModel = await this.findNextModel(resultContext);
-				if (nextModel) {
+				if (isGenerateResult(result)) {
 					/**
-					* Set the model for the next attempt
+					* Check if the result should trigger a retry
 					*/
-					this.currentModel = nextModel;
+					const resultAttempt = {
+						type: "result",
+						result,
+						model: this.currentModel
+					};
 					/**
-					* Set the previous attempt that triggered this retry
+					* Add the current attempt to the list before checking for retries
 					*/
-					previousAttempt = resultAttempt;
+					attempts.push(resultAttempt);
+					const resultContext = {
+						current: resultAttempt,
+						attempts,
+						totalAttempts
+					};
+					const nextModel = await this.findNextModel(resultContext);
+					if (nextModel) {
+						/**
+						* Set the model for the next attempt
+						*/
+						this.currentModel = nextModel;
+						/**
+						* Set the previous attempt that triggered this retry
+						*/
+						previousAttempt = resultAttempt;
+						/**
+						* Continue to the next iteration to retry
+						*/
+						continue;
+					}
 					/**
-					* Continue to the next iteration to retry
+					* No retry needed, remove the attempt since it was successful
 					*/
-					continue;
+					attempts.pop();
 				}
-				/**
-				* No retry needed, remove the attempt since it was successful and return the result
-				*/
-				attempts.pop();
 				return result;
 			} catch (error) {
 				/**
@@ -187,15 +201,7 @@ var RetryableModel = class {
 				* If we retried the request, wrap the error into a `RetryError` for better visibility.
 				*/
 				if (!nextModel) {
-					if (totalAttempts > 1) {
-						const errorMessage = getErrorMessage(error);
-						const errors = attempts.flatMap((a) => isErrorAttempt(a) ? a.error : `Result with finishReason: ${a.result.finishReason}`);
-						throw new RetryError({
-							message: `Failed after ${totalAttempts} attempts. Last error: ${errorMessage}`,
-							reason: "maxRetriesExceeded",
-							errors
-						});
-					}
+					if (totalAttempts > 1) throw this.prepareRetryError(error, attempts);
 					throw error;
 				}
 				/**
@@ -209,8 +215,20 @@ var RetryableModel = class {
 			}
 		}
 	}
+	async doGenerate(options) {
+		return this.executeWithRetry(async () => await this.currentModel.doGenerate(options));
+	}
 	async doStream(options) {
-		throw new Error("Streaming not implemented");
+		return this.executeWithRetry(async () => await this.currentModel.doStream(options));
+	}
+	prepareRetryError(error, attempts) {
+		const errorMessage = getErrorMessage(error);
+		const errors = attempts.flatMap((a) => isErrorAttempt(a) ? a.error : `Result with finishReason: ${a.result.finishReason}`);
+		return new RetryError(new RetryError({
+			message: `Failed after ${attempts.length} attempts. Last error: ${errorMessage}`,
+			reason: "maxRetriesExceeded",
+			errors
+		}));
 	}
 };
 function createRetryable(config) {
@@ -218,4 +236,4 @@ function createRetryable(config) {
 }
 
 //#endregion
-export { createRetryable, getModelKey, isErrorAttempt, isResultAttempt };
+export { createRetryable, getModelKey, isErrorAttempt, isObject, isResultAttempt, isString };

package/dist/index.js

@@ -1,3 +1,3 @@
-import { createRetryable, getModelKey, isErrorAttempt, isResultAttempt } from "./create-retryable-model-CnrFowSg.js";
+import { createRetryable, getModelKey, isErrorAttempt, isResultAttempt } from "./create-retryable-model-C4nAHxnW.js";
 
 export { createRetryable, getModelKey, isErrorAttempt, isResultAttempt };

package/dist/retryables/index.d.ts

@@ -1,8 +1,31 @@
 import { RetryModel, Retryable } from "../create-retryable-model-DzDFqgQO.js";
 import { LanguageModelV2 } from "@ai-sdk/provider";
 
-//#region src/retryables/content-filter-triggered.d.ts
+//#region src/retryables/anthropic-service-overloaded.d.ts
 
+/**
+ * Type for Anthropic error responses.
+ *
+ * @see https://docs.claude.com/en/api/errors#error-shapes
+ */
+type AnthropicErrorResponse = {
+  type: 'error';
+  error: {
+    type: string;
+    message: string;
+  };
+};
+/**
+ * Fallback if Anthropic returns an "overloaded" error with HTTP 200.
+ *
+ * ```
+ * HTTP 200 OK
+ * {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}
+ * ```
+ */
+declare function anthropicServiceOverloaded(model: LanguageModelV2, options?: Omit<RetryModel, 'model'>): Retryable;
+//#endregion
+//#region src/retryables/content-filter-triggered.d.ts
 /**
  * Fallback to a different model if the content filter was triggered.
  */
@@ -21,7 +44,10 @@ declare function requestNotRetryable(model: LanguageModelV2, options?: Omit<Retr
  */
 declare function requestTimeout(model: LanguageModelV2, options?: Omit<RetryModel, 'model'>): Retryable;
 //#endregion
-//#region src/retryables/response-schema-mismatch.d.ts
-declare function responseSchemaMismatch(model: LanguageModelV2, options?: Omit<RetryModel, 'model'>): Retryable;
+//#region src/retryables/service-overloaded.d.ts
+/**
+ * Fallback to a different model if the provider returns a HTTP 529 error.
+ */
+declare function serviceOverloaded(model: LanguageModelV2, options?: Omit<RetryModel, 'model'>): Retryable;
 //#endregion
-export { contentFilterTriggered, requestNotRetryable, requestTimeout, responseSchemaMismatch };
+export { AnthropicErrorResponse, anthropicServiceOverloaded, contentFilterTriggered, requestNotRetryable, requestTimeout, serviceOverloaded };

package/dist/retryables/index.js

@@ -1,10 +1,37 @@
-import { isErrorAttempt, isResultAttempt } from "../create-retryable-model-CnrFowSg.js";
+import { isErrorAttempt, isObject, isResultAttempt, isString } from "../create-retryable-model-C4nAHxnW.js";
 import { isAbortError } from "@ai-sdk/provider-utils";
-import { APICallError, NoObjectGeneratedError, TypeValidationError } from "ai";
+import { APICallError } from "ai";
 
-//#region src/utils.ts
-const isObject = (value) => typeof value === "object" && value !== null;
-const isString = (value) => typeof value === "string";
+//#region src/retryables/anthropic-service-overloaded.ts
+/**
+* Fallback if Anthropic returns an "overloaded" error with HTTP 200.
+*
+* ```
+* HTTP 200 OK
+* {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}
+* ```
+*/
+function anthropicServiceOverloaded(model, options) {
+	return (context) => {
+		const { current } = context;
+		if (isErrorAttempt(current)) {
+			const { error } = current;
+			if (APICallError.isInstance(error) && error.statusCode === 529) return {
+				model,
+				maxAttempts: 1,
+				...options
+			};
+			if (APICallError.isInstance(error) && error.statusCode === 200) try {
+				const responseBody = JSON.parse(error.responseBody ?? "");
+				if (responseBody.error && isObject(responseBody.error) && isString(responseBody.error.type) && responseBody.error.type === "overloaded_error") return {
+					model,
+					maxAttempts: 1,
+					...options
+				};
+			} catch {}
+		}
+	};
+}
 
 //#endregion
 //#region src/retryables/content-filter-triggered.ts
@@ -74,12 +101,16 @@ function requestTimeout(model, options) {
 }
 
 //#endregion
-//#region src/retryables/response-schema-mismatch.ts
-function responseSchemaMismatch(model, options) {
+//#region src/retryables/service-overloaded.ts
+/**
+* Fallback to a different model if the provider returns a HTTP 529 error.
+*/
+function serviceOverloaded(model, options) {
 	return (context) => {
 		const { current } = context;
 		if (isErrorAttempt(current)) {
-			if (NoObjectGeneratedError.isInstance(current.error) && current.error.finishReason === "stop" && TypeValidationError.isInstance(current.error.cause)) return {
+			const { error } = current;
+			if (APICallError.isInstance(error) && error.statusCode === 529) return {
 				model,
 				maxAttempts: 1,
 				...options
@@ -89,4 +120,4 @@ function responseSchemaMismatch(model, options) {
 }
 
 //#endregion
-export { contentFilterTriggered, requestNotRetryable, requestTimeout, responseSchemaMismatch };
+export { anthropicServiceOverloaded, contentFilterTriggered, requestNotRetryable, requestTimeout, serviceOverloaded };

package/package.json

@@ -1,7 +1,8 @@
 {
   "name": "ai-retry",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "description": "AI SDK Retry",
+  "packageManager": "pnpm@9.0.0",
   "main": "./dist/index.js",
   "module": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -17,6 +18,14 @@
   "publishConfig": {
     "access": "public"
   },
+  "scripts": {
+    "prepublishOnly": "pnpm build",
+    "publish:alpha": "pnpm version prerelease --preid alpha && pnpm publish --tag alpha",
+    "build": "tsdown",
+    "test": "vitest",
+    "lint": "biome check . --write",
+    "prepare": "husky"
+  },
   "keywords": [
     "ai",
     "ai-sdk",
@@ -33,6 +42,7 @@
     "ai": "5.x"
   },
   "devDependencies": {
+    "@ai-sdk/anthropic": "^2.0.18",
     "@ai-sdk/azure": "^2.0.30",
     "@ai-sdk/openai": "^2.0.30",
     "@arethetypeswrong/cli": "^0.18.2",
@@ -54,11 +64,5 @@
   "dependencies": {
     "@ai-sdk/provider": "^2.0.0",
     "@ai-sdk/provider-utils": "^3.0.9"
-  },
-  "scripts": {
-    "publish:alpha": "pnpm version prerelease --preid alpha && pnpm publish --tag alpha",
-    "build": "tsdown",
-    "test": "vitest",
-    "lint": "biome check . --write"
   }
-}
+}