@ai-sdk/replicate 0.0.0-1c33ba03-20260114162300 → 0.0.0-4115c213-20260122152721
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -3
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/docs/60-replicate.mdx +241 -0
- package/package.json +14 -5
- package/src/index.ts +7 -0
- package/src/replicate-error.ts +13 -0
- package/src/replicate-image-model.ts +268 -0
- package/src/replicate-image-settings.ts +36 -0
- package/src/replicate-provider.ts +99 -0
- package/src/version.ts +6 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,11 +1,40 @@
|
|
|
1
1
|
# @ai-sdk/replicate
|
|
2
2
|
|
|
3
|
-
## 0.0.0-
|
|
3
|
+
## 0.0.0-4115c213-20260122152721
|
|
4
4
|
|
|
5
5
|
### Patch Changes
|
|
6
6
|
|
|
7
|
-
-
|
|
8
|
-
|
|
7
|
+
- 4caafb2: chore: excluded tests from src folder in npm package
|
|
8
|
+
- Updated dependencies [4caafb2]
|
|
9
|
+
- @ai-sdk/provider@0.0.0-4115c213-20260122152721
|
|
10
|
+
- @ai-sdk/provider-utils@0.0.0-4115c213-20260122152721
|
|
11
|
+
|
|
12
|
+
## 2.0.10
|
|
13
|
+
|
|
14
|
+
### Patch Changes
|
|
15
|
+
|
|
16
|
+
- 2b8369d: chore: add docs to package dist
|
|
17
|
+
|
|
18
|
+
## 2.0.9
|
|
19
|
+
|
|
20
|
+
### Patch Changes
|
|
21
|
+
|
|
22
|
+
- 8dc54db: chore: add src folders to package bundle
|
|
23
|
+
|
|
24
|
+
## 2.0.8
|
|
25
|
+
|
|
26
|
+
### Patch Changes
|
|
27
|
+
|
|
28
|
+
- Updated dependencies [5c090e7]
|
|
29
|
+
- @ai-sdk/provider@3.0.4
|
|
30
|
+
- @ai-sdk/provider-utils@4.0.8
|
|
31
|
+
|
|
32
|
+
## 2.0.7
|
|
33
|
+
|
|
34
|
+
### Patch Changes
|
|
35
|
+
|
|
36
|
+
- Updated dependencies [46f46e4]
|
|
37
|
+
- @ai-sdk/provider-utils@4.0.7
|
|
9
38
|
|
|
10
39
|
## 2.0.6
|
|
11
40
|
|
package/dist/index.js
CHANGED
|
@@ -227,7 +227,7 @@ var replicateImageProviderOptionsSchema = (0, import_provider_utils2.lazySchema)
|
|
|
227
227
|
);
|
|
228
228
|
|
|
229
229
|
// src/version.ts
|
|
230
|
-
var VERSION = true ? "0.0.0-
|
|
230
|
+
var VERSION = true ? "0.0.0-4115c213-20260122152721" : "0.0.0-test";
|
|
231
231
|
|
|
232
232
|
// src/replicate-provider.ts
|
|
233
233
|
function createReplicate(options = {}) {
|
package/dist/index.mjs
CHANGED
|
@@ -210,7 +210,7 @@ var replicateImageProviderOptionsSchema = lazySchema(
|
|
|
210
210
|
);
|
|
211
211
|
|
|
212
212
|
// src/version.ts
|
|
213
|
-
var VERSION = true ? "0.0.0-
|
|
213
|
+
var VERSION = true ? "0.0.0-4115c213-20260122152721" : "0.0.0-test";
|
|
214
214
|
|
|
215
215
|
// src/replicate-provider.ts
|
|
216
216
|
function createReplicate(options = {}) {
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Replicate
|
|
3
|
+
description: Learn how to use Replicate models with the AI SDK.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Replicate Provider
|
|
7
|
+
|
|
8
|
+
[Replicate](https://replicate.com/) is a platform for running open-source AI models.
|
|
9
|
+
It is a popular choice for running image generation models.
|
|
10
|
+
|
|
11
|
+
## Setup
|
|
12
|
+
|
|
13
|
+
The Replicate provider is available via the `@ai-sdk/replicate` module. You can install it with
|
|
14
|
+
|
|
15
|
+
<Tabs items={['pnpm', 'npm', 'yarn', 'bun']}>
|
|
16
|
+
<Tab>
|
|
17
|
+
<Snippet text="pnpm add @ai-sdk/replicate" dark />
|
|
18
|
+
</Tab>
|
|
19
|
+
<Tab>
|
|
20
|
+
<Snippet text="npm install @ai-sdk/replicate" dark />
|
|
21
|
+
</Tab>
|
|
22
|
+
<Tab>
|
|
23
|
+
<Snippet text="yarn add @ai-sdk/replicate" dark />
|
|
24
|
+
</Tab>
|
|
25
|
+
|
|
26
|
+
<Tab>
|
|
27
|
+
<Snippet text="bun add @ai-sdk/replicate" dark />
|
|
28
|
+
</Tab>
|
|
29
|
+
</Tabs>
|
|
30
|
+
|
|
31
|
+
## Provider Instance
|
|
32
|
+
|
|
33
|
+
You can import the default provider instance `replicate` from `@ai-sdk/replicate`:
|
|
34
|
+
|
|
35
|
+
```ts
|
|
36
|
+
import { replicate } from '@ai-sdk/replicate';
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
If you need a customized setup, you can import `createReplicate` from `@ai-sdk/replicate`
|
|
40
|
+
and create a provider instance with your settings:
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import { createReplicate } from '@ai-sdk/replicate';
|
|
44
|
+
|
|
45
|
+
const replicate = createReplicate({
|
|
46
|
+
apiToken: process.env.REPLICATE_API_TOKEN ?? '',
|
|
47
|
+
});
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
You can use the following optional settings to customize the Replicate provider instance:
|
|
51
|
+
|
|
52
|
+
- **baseURL** _string_
|
|
53
|
+
|
|
54
|
+
Use a different URL prefix for API calls, e.g. to use proxy servers.
|
|
55
|
+
The default prefix is `https://api.replicate.com/v1`.
|
|
56
|
+
|
|
57
|
+
- **apiToken** _string_
|
|
58
|
+
|
|
59
|
+
API token that is being sent using the `Authorization` header. It defaults to
|
|
60
|
+
the `REPLICATE_API_TOKEN` environment variable.
|
|
61
|
+
|
|
62
|
+
- **headers** _Record<string,string>_
|
|
63
|
+
|
|
64
|
+
Custom headers to include in the requests.
|
|
65
|
+
|
|
66
|
+
- **fetch** _(input: RequestInfo, init?: RequestInit) => Promise<Response>_
|
|
67
|
+
|
|
68
|
+
Custom [fetch](https://developer.mozilla.org/en-US/docs/Web/API/fetch) implementation.
|
|
69
|
+
|
|
70
|
+
## Image Models
|
|
71
|
+
|
|
72
|
+
You can create Replicate image models using the `.image()` factory method.
|
|
73
|
+
For more on image generation with the AI SDK see [generateImage()](/docs/reference/ai-sdk-core/generate-image).
|
|
74
|
+
|
|
75
|
+
<Note>
|
|
76
|
+
Model support for `size` and other parameters varies by model. Check the
|
|
77
|
+
model's documentation on [Replicate](https://replicate.com/explore) for
|
|
78
|
+
supported options and additional parameters that can be passed via
|
|
79
|
+
`providerOptions.replicate`.
|
|
80
|
+
</Note>
|
|
81
|
+
|
|
82
|
+
### Supported Image Models
|
|
83
|
+
|
|
84
|
+
The following image models are currently supported by the Replicate provider:
|
|
85
|
+
|
|
86
|
+
- [black-forest-labs/flux-1.1-pro-ultra](https://replicate.com/black-forest-labs/flux-1.1-pro-ultra)
|
|
87
|
+
- [black-forest-labs/flux-1.1-pro](https://replicate.com/black-forest-labs/flux-1.1-pro)
|
|
88
|
+
- [black-forest-labs/flux-dev](https://replicate.com/black-forest-labs/flux-dev)
|
|
89
|
+
- [black-forest-labs/flux-pro](https://replicate.com/black-forest-labs/flux-pro)
|
|
90
|
+
- [black-forest-labs/flux-schnell](https://replicate.com/black-forest-labs/flux-schnell)
|
|
91
|
+
- [bytedance/sdxl-lightning-4step](https://replicate.com/bytedance/sdxl-lightning-4step)
|
|
92
|
+
- [fofr/aura-flow](https://replicate.com/fofr/aura-flow)
|
|
93
|
+
- [fofr/latent-consistency-model](https://replicate.com/fofr/latent-consistency-model)
|
|
94
|
+
- [fofr/realvisxl-v3-multi-controlnet-lora](https://replicate.com/fofr/realvisxl-v3-multi-controlnet-lora)
|
|
95
|
+
- [fofr/sdxl-emoji](https://replicate.com/fofr/sdxl-emoji)
|
|
96
|
+
- [fofr/sdxl-multi-controlnet-lora](https://replicate.com/fofr/sdxl-multi-controlnet-lora)
|
|
97
|
+
- [ideogram-ai/ideogram-v2-turbo](https://replicate.com/ideogram-ai/ideogram-v2-turbo)
|
|
98
|
+
- [ideogram-ai/ideogram-v2](https://replicate.com/ideogram-ai/ideogram-v2)
|
|
99
|
+
- [lucataco/dreamshaper-xl-turbo](https://replicate.com/lucataco/dreamshaper-xl-turbo)
|
|
100
|
+
- [lucataco/open-dalle-v1.1](https://replicate.com/lucataco/open-dalle-v1.1)
|
|
101
|
+
- [lucataco/realvisxl-v2.0](https://replicate.com/lucataco/realvisxl-v2.0)
|
|
102
|
+
- [lucataco/realvisxl2-lcm](https://replicate.com/lucataco/realvisxl2-lcm)
|
|
103
|
+
- [luma/photon-flash](https://replicate.com/luma/photon-flash)
|
|
104
|
+
- [luma/photon](https://replicate.com/luma/photon)
|
|
105
|
+
- [nvidia/sana](https://replicate.com/nvidia/sana)
|
|
106
|
+
- [playgroundai/playground-v2.5-1024px-aesthetic](https://replicate.com/playgroundai/playground-v2.5-1024px-aesthetic)
|
|
107
|
+
- [recraft-ai/recraft-v3-svg](https://replicate.com/recraft-ai/recraft-v3-svg)
|
|
108
|
+
- [recraft-ai/recraft-v3](https://replicate.com/recraft-ai/recraft-v3)
|
|
109
|
+
- [stability-ai/stable-diffusion-3.5-large-turbo](https://replicate.com/stability-ai/stable-diffusion-3.5-large-turbo)
|
|
110
|
+
- [stability-ai/stable-diffusion-3.5-large](https://replicate.com/stability-ai/stable-diffusion-3.5-large)
|
|
111
|
+
- [stability-ai/stable-diffusion-3.5-medium](https://replicate.com/stability-ai/stable-diffusion-3.5-medium)
|
|
112
|
+
- [tstramer/material-diffusion](https://replicate.com/tstramer/material-diffusion)
|
|
113
|
+
|
|
114
|
+
You can also use [versioned models](https://replicate.com/docs/topics/models/versions).
|
|
115
|
+
The id for versioned models is the Replicate model id followed by a colon and the version ID (`$modelId:$versionId`), e.g.
|
|
116
|
+
`bytedance/sdxl-lightning-4step:5599ed30703defd1d160a25a63321b4dec97101d98b4674bcc56e41f62f35637`.
|
|
117
|
+
|
|
118
|
+
<Note>
|
|
119
|
+
You can also pass any available Replicate model ID as a string if needed.
|
|
120
|
+
</Note>
|
|
121
|
+
|
|
122
|
+
### Basic Usage
|
|
123
|
+
|
|
124
|
+
```ts
|
|
125
|
+
import { replicate } from '@ai-sdk/replicate';
|
|
126
|
+
import { generateImage } from 'ai';
|
|
127
|
+
import { writeFile } from 'node:fs/promises';
|
|
128
|
+
|
|
129
|
+
const { image } = await generateImage({
|
|
130
|
+
model: replicate.image('black-forest-labs/flux-schnell'),
|
|
131
|
+
prompt: 'The Loch Ness Monster getting a manicure',
|
|
132
|
+
aspectRatio: '16:9',
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
await writeFile('image.webp', image.uint8Array);
|
|
136
|
+
|
|
137
|
+
console.log('Image saved as image.webp');
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### Model-specific options
|
|
141
|
+
|
|
142
|
+
```ts highlight="9-11"
|
|
143
|
+
import { replicate } from '@ai-sdk/replicate';
|
|
144
|
+
import { generateImage } from 'ai';
|
|
145
|
+
|
|
146
|
+
const { image } = await generateImage({
|
|
147
|
+
model: replicate.image('recraft-ai/recraft-v3'),
|
|
148
|
+
prompt: 'The Loch Ness Monster getting a manicure',
|
|
149
|
+
size: '1365x1024',
|
|
150
|
+
providerOptions: {
|
|
151
|
+
replicate: {
|
|
152
|
+
style: 'realistic_image',
|
|
153
|
+
},
|
|
154
|
+
},
|
|
155
|
+
});
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
### Versioned Models
|
|
159
|
+
|
|
160
|
+
```ts
|
|
161
|
+
import { replicate } from '@ai-sdk/replicate';
|
|
162
|
+
import { generateImage } from 'ai';
|
|
163
|
+
|
|
164
|
+
const { image } = await generateImage({
|
|
165
|
+
model: replicate.image(
|
|
166
|
+
'bytedance/sdxl-lightning-4step:5599ed30703defd1d160a25a63321b4dec97101d98b4674bcc56e41f62f35637',
|
|
167
|
+
),
|
|
168
|
+
prompt: 'The Loch Ness Monster getting a manicure',
|
|
169
|
+
});
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
### Image Editing
|
|
173
|
+
|
|
174
|
+
Replicate supports image editing through various models. Pass input images via `prompt.images` to transform or edit existing images.
|
|
175
|
+
|
|
176
|
+
#### Basic Image Editing
|
|
177
|
+
|
|
178
|
+
Transform an existing image using text prompts:
|
|
179
|
+
|
|
180
|
+
```ts
|
|
181
|
+
const imageBuffer = readFileSync('./input-image.png');
|
|
182
|
+
|
|
183
|
+
const { images } = await generateImage({
|
|
184
|
+
model: replicate.image('black-forest-labs/flux-kontext-dev'),
|
|
185
|
+
prompt: {
|
|
186
|
+
text: 'Turn the cat into a golden retriever dog',
|
|
187
|
+
images: [imageBuffer],
|
|
188
|
+
},
|
|
189
|
+
providerOptions: {
|
|
190
|
+
replicate: {
|
|
191
|
+
guidance_scale: 7.5,
|
|
192
|
+
num_inference_steps: 30,
|
|
193
|
+
},
|
|
194
|
+
},
|
|
195
|
+
});
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
#### Inpainting with Mask
|
|
199
|
+
|
|
200
|
+
Edit specific parts of an image using a mask. For FLUX Fill models, white areas in the mask indicate where the image should be edited:
|
|
201
|
+
|
|
202
|
+
```ts
|
|
203
|
+
const image = readFileSync('./input-image.png');
|
|
204
|
+
const mask = readFileSync('./mask.png'); // White = inpaint, black = keep
|
|
205
|
+
|
|
206
|
+
const { images } = await generateImage({
|
|
207
|
+
model: replicate.image('black-forest-labs/flux-fill-pro'),
|
|
208
|
+
prompt: {
|
|
209
|
+
text: 'A sunlit indoor lounge area with a pool containing a flamingo',
|
|
210
|
+
images: [image],
|
|
211
|
+
mask: mask,
|
|
212
|
+
},
|
|
213
|
+
providerOptions: {
|
|
214
|
+
replicate: {
|
|
215
|
+
guidance_scale: 7.5,
|
|
216
|
+
num_inference_steps: 30,
|
|
217
|
+
},
|
|
218
|
+
},
|
|
219
|
+
});
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
<Note>
|
|
223
|
+
Input images can be provided as `Buffer`, `ArrayBuffer`, `Uint8Array`, or
|
|
224
|
+
base64-encoded strings. Different Replicate models have different parameter
|
|
225
|
+
names and capabilities — check the model's documentation on
|
|
226
|
+
[Replicate](https://replicate.com/explore) for details.
|
|
227
|
+
</Note>
|
|
228
|
+
|
|
229
|
+
### Provider Options
|
|
230
|
+
|
|
231
|
+
Common provider options for image generation:
|
|
232
|
+
|
|
233
|
+
- **maxWaitTimeInSeconds** _number_ - Maximum time in seconds to wait for the prediction to complete in sync mode. By default, Replicate uses [sync mode](https://replicate.com/docs/topics/predictions/create-a-prediction#timeout-duration) with a 60-second timeout. Set to a positive number to use a custom duration (e.g., `120` for 2 minutes). When not specified, uses the default 60-second wait.
|
|
234
|
+
- **guidance_scale** _number_ - Guidance scale for classifier-free guidance. Higher values make the output more closely match the prompt.
|
|
235
|
+
- **num_inference_steps** _number_ - Number of denoising steps. More steps = higher quality but slower.
|
|
236
|
+
- **negative_prompt** _string_ - Negative prompt to guide what to avoid in the generation.
|
|
237
|
+
- **output_format** _'png' | 'jpg' | 'webp'_ - Output image format.
|
|
238
|
+
- **output_quality** _number (1-100)_ - Output image quality. Only applies to jpg and webp.
|
|
239
|
+
- **strength** _number (0-1)_ - Strength of the transformation for img2img. Lower values keep more of the original image.
|
|
240
|
+
|
|
241
|
+
For more details, see the [Replicate models page](https://replicate.com/explore).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ai-sdk/replicate",
|
|
3
|
-
"version": "0.0.0-
|
|
3
|
+
"version": "0.0.0-4115c213-20260122152721",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -8,9 +8,18 @@
|
|
|
8
8
|
"types": "./dist/index.d.ts",
|
|
9
9
|
"files": [
|
|
10
10
|
"dist/**/*",
|
|
11
|
+
"docs/**/*",
|
|
12
|
+
"src",
|
|
13
|
+
"!src/**/*.test.ts",
|
|
14
|
+
"!src/**/*.test-d.ts",
|
|
15
|
+
"!src/**/__snapshots__",
|
|
16
|
+
"!src/**/__fixtures__",
|
|
11
17
|
"CHANGELOG.md",
|
|
12
18
|
"README.md"
|
|
13
19
|
],
|
|
20
|
+
"directories": {
|
|
21
|
+
"doc": "./docs"
|
|
22
|
+
},
|
|
14
23
|
"exports": {
|
|
15
24
|
"./package.json": "./package.json",
|
|
16
25
|
".": {
|
|
@@ -20,15 +29,15 @@
|
|
|
20
29
|
}
|
|
21
30
|
},
|
|
22
31
|
"dependencies": {
|
|
23
|
-
"@ai-sdk/provider": "
|
|
24
|
-
"@ai-sdk/provider-utils": "0.0.0-
|
|
32
|
+
"@ai-sdk/provider": "0.0.0-4115c213-20260122152721",
|
|
33
|
+
"@ai-sdk/provider-utils": "0.0.0-4115c213-20260122152721"
|
|
25
34
|
},
|
|
26
35
|
"devDependencies": {
|
|
27
36
|
"@types/node": "20.17.24",
|
|
28
37
|
"tsup": "^8",
|
|
29
38
|
"typescript": "5.8.3",
|
|
30
39
|
"zod": "3.25.76",
|
|
31
|
-
"@ai-sdk/test-server": "
|
|
40
|
+
"@ai-sdk/test-server": "0.0.0-4115c213-20260122152721",
|
|
32
41
|
"@vercel/ai-tsconfig": "0.0.0"
|
|
33
42
|
},
|
|
34
43
|
"peerDependencies": {
|
|
@@ -54,7 +63,7 @@
|
|
|
54
63
|
"scripts": {
|
|
55
64
|
"build": "pnpm clean && tsup --tsconfig tsconfig.build.json",
|
|
56
65
|
"build:watch": "pnpm clean && tsup --watch",
|
|
57
|
-
"clean": "del-cli dist *.tsbuildinfo",
|
|
66
|
+
"clean": "del-cli dist docs *.tsbuildinfo",
|
|
58
67
|
"lint": "eslint \"./**/*.ts*\"",
|
|
59
68
|
"type-check": "tsc --build",
|
|
60
69
|
"prettier-check": "prettier --check \"./**/*.ts*\"",
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { createReplicate, replicate } from './replicate-provider';
|
|
2
|
+
export type {
|
|
3
|
+
ReplicateProvider,
|
|
4
|
+
ReplicateProviderSettings,
|
|
5
|
+
} from './replicate-provider';
|
|
6
|
+
export type { ReplicateImageProviderOptions } from './replicate-image-model';
|
|
7
|
+
export { VERSION } from './version';
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { createJsonErrorResponseHandler } from '@ai-sdk/provider-utils';
|
|
2
|
+
import { z } from 'zod/v4';
|
|
3
|
+
|
|
4
|
+
const replicateErrorSchema = z.object({
|
|
5
|
+
detail: z.string().optional(),
|
|
6
|
+
error: z.string().optional(),
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
export const replicateFailedResponseHandler = createJsonErrorResponseHandler({
|
|
10
|
+
errorSchema: replicateErrorSchema,
|
|
11
|
+
errorToMessage: error =>
|
|
12
|
+
error.detail ?? error.error ?? 'Unknown Replicate error',
|
|
13
|
+
});
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
import type { ImageModelV3, SharedV3Warning } from '@ai-sdk/provider';
|
|
2
|
+
import type { Resolvable } from '@ai-sdk/provider-utils';
|
|
3
|
+
import {
|
|
4
|
+
combineHeaders,
|
|
5
|
+
convertImageModelFileToDataUri,
|
|
6
|
+
createBinaryResponseHandler,
|
|
7
|
+
createJsonResponseHandler,
|
|
8
|
+
FetchFunction,
|
|
9
|
+
getFromApi,
|
|
10
|
+
InferSchema,
|
|
11
|
+
lazySchema,
|
|
12
|
+
parseProviderOptions,
|
|
13
|
+
postJsonToApi,
|
|
14
|
+
resolve,
|
|
15
|
+
zodSchema,
|
|
16
|
+
} from '@ai-sdk/provider-utils';
|
|
17
|
+
import { z } from 'zod/v4';
|
|
18
|
+
import { replicateFailedResponseHandler } from './replicate-error';
|
|
19
|
+
import { ReplicateImageModelId } from './replicate-image-settings';
|
|
20
|
+
|
|
21
|
+
interface ReplicateImageModelConfig {
|
|
22
|
+
provider: string;
|
|
23
|
+
baseURL: string;
|
|
24
|
+
headers?: Resolvable<Record<string, string | undefined>>;
|
|
25
|
+
fetch?: FetchFunction;
|
|
26
|
+
_internal?: {
|
|
27
|
+
currentDate?: () => Date;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Flux-2 models support up to 8 input images with input_image, input_image_2, etc.
|
|
32
|
+
const FLUX_2_MODEL_PATTERN = /^black-forest-labs\/flux-2-/;
|
|
33
|
+
const MAX_FLUX_2_INPUT_IMAGES = 8;
|
|
34
|
+
|
|
35
|
+
export class ReplicateImageModel implements ImageModelV3 {
|
|
36
|
+
readonly specificationVersion = 'v3';
|
|
37
|
+
|
|
38
|
+
get maxImagesPerCall(): number {
|
|
39
|
+
// Flux-2 models support up to 8 input images
|
|
40
|
+
return this.isFlux2Model ? MAX_FLUX_2_INPUT_IMAGES : 1;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
get provider(): string {
|
|
44
|
+
return this.config.provider;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
private get isFlux2Model(): boolean {
|
|
48
|
+
return FLUX_2_MODEL_PATTERN.test(this.modelId);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
constructor(
|
|
52
|
+
readonly modelId: ReplicateImageModelId,
|
|
53
|
+
private readonly config: ReplicateImageModelConfig,
|
|
54
|
+
) {}
|
|
55
|
+
|
|
56
|
+
async doGenerate({
|
|
57
|
+
prompt,
|
|
58
|
+
n,
|
|
59
|
+
aspectRatio,
|
|
60
|
+
size,
|
|
61
|
+
seed,
|
|
62
|
+
providerOptions,
|
|
63
|
+
headers,
|
|
64
|
+
abortSignal,
|
|
65
|
+
files,
|
|
66
|
+
mask,
|
|
67
|
+
}: Parameters<ImageModelV3['doGenerate']>[0]): Promise<
|
|
68
|
+
Awaited<ReturnType<ImageModelV3['doGenerate']>>
|
|
69
|
+
> {
|
|
70
|
+
const warnings: Array<SharedV3Warning> = [];
|
|
71
|
+
|
|
72
|
+
const [modelId, version] = this.modelId.split(':');
|
|
73
|
+
|
|
74
|
+
const currentDate = this.config._internal?.currentDate?.() ?? new Date();
|
|
75
|
+
|
|
76
|
+
// Parse provider options
|
|
77
|
+
const replicateOptions = await parseProviderOptions({
|
|
78
|
+
provider: 'replicate',
|
|
79
|
+
providerOptions,
|
|
80
|
+
schema: replicateImageProviderOptionsSchema,
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
// Handle image input from files
|
|
84
|
+
let imageInputs: Record<string, string> = {};
|
|
85
|
+
if (files != null && files.length > 0) {
|
|
86
|
+
if (this.isFlux2Model) {
|
|
87
|
+
// Flux-2 models use input_image, input_image_2, input_image_3, etc.
|
|
88
|
+
for (
|
|
89
|
+
let i = 0;
|
|
90
|
+
i < Math.min(files.length, MAX_FLUX_2_INPUT_IMAGES);
|
|
91
|
+
i++
|
|
92
|
+
) {
|
|
93
|
+
const key = i === 0 ? 'input_image' : `input_image_${i + 1}`;
|
|
94
|
+
imageInputs[key] = convertImageModelFileToDataUri(files[i]);
|
|
95
|
+
}
|
|
96
|
+
if (files.length > MAX_FLUX_2_INPUT_IMAGES) {
|
|
97
|
+
warnings.push({
|
|
98
|
+
type: 'other',
|
|
99
|
+
message: `Flux-2 models support up to ${MAX_FLUX_2_INPUT_IMAGES} input images. Additional images are ignored.`,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
} else {
|
|
103
|
+
// Other models use single 'image' parameter
|
|
104
|
+
imageInputs = { image: convertImageModelFileToDataUri(files[0]) };
|
|
105
|
+
if (files.length > 1) {
|
|
106
|
+
warnings.push({
|
|
107
|
+
type: 'other',
|
|
108
|
+
message:
|
|
109
|
+
'This Replicate model only supports a single input image. Additional images are ignored.',
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Handle mask input (not supported by Flux-2 models)
|
|
116
|
+
let maskInput: string | undefined;
|
|
117
|
+
if (mask != null) {
|
|
118
|
+
if (this.isFlux2Model) {
|
|
119
|
+
warnings.push({
|
|
120
|
+
type: 'other',
|
|
121
|
+
message:
|
|
122
|
+
'Flux-2 models do not support mask input. The mask will be ignored.',
|
|
123
|
+
});
|
|
124
|
+
} else {
|
|
125
|
+
maskInput = convertImageModelFileToDataUri(mask);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Extract maxWaitTimeInSeconds from provider options and prepare the rest for the request body
|
|
130
|
+
const { maxWaitTimeInSeconds, ...inputOptions } = replicateOptions ?? {};
|
|
131
|
+
|
|
132
|
+
// Build the prefer header based on maxWaitTimeInSeconds:
|
|
133
|
+
// - undefined/null: use default sync wait (prefer: wait)
|
|
134
|
+
// - positive number: use custom wait duration (prefer: wait=N)
|
|
135
|
+
const preferHeader: Record<string, string> =
|
|
136
|
+
maxWaitTimeInSeconds != null
|
|
137
|
+
? { prefer: `wait=${maxWaitTimeInSeconds}` }
|
|
138
|
+
: { prefer: 'wait' };
|
|
139
|
+
|
|
140
|
+
const {
|
|
141
|
+
value: { output },
|
|
142
|
+
responseHeaders,
|
|
143
|
+
} = await postJsonToApi({
|
|
144
|
+
url:
|
|
145
|
+
// different endpoints for versioned vs unversioned models:
|
|
146
|
+
version != null
|
|
147
|
+
? `${this.config.baseURL}/predictions`
|
|
148
|
+
: `${this.config.baseURL}/models/${modelId}/predictions`,
|
|
149
|
+
|
|
150
|
+
headers: combineHeaders(
|
|
151
|
+
await resolve(this.config.headers),
|
|
152
|
+
headers,
|
|
153
|
+
preferHeader,
|
|
154
|
+
),
|
|
155
|
+
|
|
156
|
+
body: {
|
|
157
|
+
input: {
|
|
158
|
+
prompt,
|
|
159
|
+
aspect_ratio: aspectRatio,
|
|
160
|
+
size,
|
|
161
|
+
seed,
|
|
162
|
+
num_outputs: n,
|
|
163
|
+
...imageInputs,
|
|
164
|
+
...(maskInput != null ? { mask: maskInput } : {}),
|
|
165
|
+
...inputOptions,
|
|
166
|
+
},
|
|
167
|
+
// for versioned models, include the version in the body:
|
|
168
|
+
...(version != null ? { version } : {}),
|
|
169
|
+
},
|
|
170
|
+
|
|
171
|
+
successfulResponseHandler: createJsonResponseHandler(
|
|
172
|
+
replicateImageResponseSchema,
|
|
173
|
+
),
|
|
174
|
+
failedResponseHandler: replicateFailedResponseHandler,
|
|
175
|
+
abortSignal,
|
|
176
|
+
fetch: this.config.fetch,
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
// download the images:
|
|
180
|
+
const outputArray = Array.isArray(output) ? output : [output];
|
|
181
|
+
const images = await Promise.all(
|
|
182
|
+
outputArray.map(async url => {
|
|
183
|
+
const { value: image } = await getFromApi({
|
|
184
|
+
url,
|
|
185
|
+
successfulResponseHandler: createBinaryResponseHandler(),
|
|
186
|
+
failedResponseHandler: replicateFailedResponseHandler,
|
|
187
|
+
abortSignal,
|
|
188
|
+
fetch: this.config.fetch,
|
|
189
|
+
});
|
|
190
|
+
return image;
|
|
191
|
+
}),
|
|
192
|
+
);
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
images,
|
|
196
|
+
warnings,
|
|
197
|
+
response: {
|
|
198
|
+
timestamp: currentDate,
|
|
199
|
+
modelId: this.modelId,
|
|
200
|
+
headers: responseHeaders,
|
|
201
|
+
},
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const replicateImageResponseSchema = z.object({
|
|
207
|
+
output: z.union([z.array(z.string()), z.string()]),
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Provider options schema for Replicate image generation.
|
|
212
|
+
*
|
|
213
|
+
* Note: Different Replicate models support different parameters.
|
|
214
|
+
* This schema includes common parameters, but you can pass any
|
|
215
|
+
* model-specific parameters through the passthrough.
|
|
216
|
+
*/
|
|
217
|
+
export const replicateImageProviderOptionsSchema = lazySchema(() =>
|
|
218
|
+
zodSchema(
|
|
219
|
+
z
|
|
220
|
+
.object({
|
|
221
|
+
/**
|
|
222
|
+
* Maximum time in seconds to wait for the prediction to complete in sync mode.
|
|
223
|
+
* By default, Replicate uses sync mode with a 60-second timeout.
|
|
224
|
+
*
|
|
225
|
+
* - When not specified: Uses default 60-second sync wait (`prefer: wait`)
|
|
226
|
+
* - When set to a positive number: Uses that duration (`prefer: wait=N`)
|
|
227
|
+
*/
|
|
228
|
+
maxWaitTimeInSeconds: z.number().positive().nullish(),
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Guidance scale for classifier-free guidance.
|
|
232
|
+
* Higher values make the output more closely match the prompt.
|
|
233
|
+
*/
|
|
234
|
+
guidance_scale: z.number().nullish(),
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Number of denoising steps. More steps = higher quality but slower.
|
|
238
|
+
*/
|
|
239
|
+
num_inference_steps: z.number().nullish(),
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Negative prompt to guide what to avoid in the generation.
|
|
243
|
+
*/
|
|
244
|
+
negative_prompt: z.string().nullish(),
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Output image format.
|
|
248
|
+
*/
|
|
249
|
+
output_format: z.enum(['png', 'jpg', 'webp']).nullish(),
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Output image quality (1-100). Only applies to jpg and webp.
|
|
253
|
+
*/
|
|
254
|
+
output_quality: z.number().min(1).max(100).nullish(),
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Strength of the transformation for img2img (0-1).
|
|
258
|
+
* Lower values keep more of the original image.
|
|
259
|
+
*/
|
|
260
|
+
strength: z.number().min(0).max(1).nullish(),
|
|
261
|
+
})
|
|
262
|
+
.passthrough(),
|
|
263
|
+
),
|
|
264
|
+
);
|
|
265
|
+
|
|
266
|
+
export type ReplicateImageProviderOptions = InferSchema<
|
|
267
|
+
typeof replicateImageProviderOptionsSchema
|
|
268
|
+
>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export type ReplicateImageModelId =
|
|
2
|
+
// Text-to-image models
|
|
3
|
+
| 'black-forest-labs/flux-1.1-pro'
|
|
4
|
+
| 'black-forest-labs/flux-1.1-pro-ultra'
|
|
5
|
+
| 'black-forest-labs/flux-dev'
|
|
6
|
+
| 'black-forest-labs/flux-pro'
|
|
7
|
+
| 'black-forest-labs/flux-schnell'
|
|
8
|
+
| 'bytedance/sdxl-lightning-4step'
|
|
9
|
+
| 'fofr/aura-flow'
|
|
10
|
+
| 'fofr/latent-consistency-model'
|
|
11
|
+
| 'fofr/realvisxl-v3-multi-controlnet-lora'
|
|
12
|
+
| 'fofr/sdxl-emoji'
|
|
13
|
+
| 'fofr/sdxl-multi-controlnet-lora'
|
|
14
|
+
| 'ideogram-ai/ideogram-v2'
|
|
15
|
+
| 'ideogram-ai/ideogram-v2-turbo'
|
|
16
|
+
| 'lucataco/dreamshaper-xl-turbo'
|
|
17
|
+
| 'lucataco/open-dalle-v1.1'
|
|
18
|
+
| 'lucataco/realvisxl-v2.0'
|
|
19
|
+
| 'lucataco/realvisxl2-lcm'
|
|
20
|
+
| 'luma/photon'
|
|
21
|
+
| 'luma/photon-flash'
|
|
22
|
+
| 'nvidia/sana'
|
|
23
|
+
| 'playgroundai/playground-v2.5-1024px-aesthetic'
|
|
24
|
+
| 'recraft-ai/recraft-v3'
|
|
25
|
+
| 'recraft-ai/recraft-v3-svg'
|
|
26
|
+
| 'stability-ai/stable-diffusion-3.5-large'
|
|
27
|
+
| 'stability-ai/stable-diffusion-3.5-large-turbo'
|
|
28
|
+
| 'stability-ai/stable-diffusion-3.5-medium'
|
|
29
|
+
| 'tstramer/material-diffusion'
|
|
30
|
+
// Inpainting and image editing models
|
|
31
|
+
| 'black-forest-labs/flux-fill-pro'
|
|
32
|
+
| 'black-forest-labs/flux-fill-dev'
|
|
33
|
+
// Flux-2 models (support up to 8 reference images)
|
|
34
|
+
| 'black-forest-labs/flux-2-pro'
|
|
35
|
+
| 'black-forest-labs/flux-2-dev'
|
|
36
|
+
| (string & {});
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { NoSuchModelError, ProviderV3 } from '@ai-sdk/provider';
|
|
2
|
+
import type { FetchFunction } from '@ai-sdk/provider-utils';
|
|
3
|
+
import { loadApiKey, withUserAgentSuffix } from '@ai-sdk/provider-utils';
|
|
4
|
+
import { ReplicateImageModel } from './replicate-image-model';
|
|
5
|
+
import { ReplicateImageModelId } from './replicate-image-settings';
|
|
6
|
+
import { VERSION } from './version';
|
|
7
|
+
|
|
8
|
+
export interface ReplicateProviderSettings {
|
|
9
|
+
/**
|
|
10
|
+
API token that is being send using the `Authorization` header.
|
|
11
|
+
It defaults to the `REPLICATE_API_TOKEN` environment variable.
|
|
12
|
+
*/
|
|
13
|
+
apiToken?: string;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
Use a different URL prefix for API calls, e.g. to use proxy servers.
|
|
17
|
+
The default prefix is `https://api.replicate.com/v1`.
|
|
18
|
+
*/
|
|
19
|
+
baseURL?: string;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
Custom headers to include in the requests.
|
|
23
|
+
*/
|
|
24
|
+
headers?: Record<string, string>;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
Custom fetch implementation. You can use it as a middleware to intercept requests,
|
|
28
|
+
or to provide a custom fetch implementation for e.g. testing.
|
|
29
|
+
*/
|
|
30
|
+
fetch?: FetchFunction;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface ReplicateProvider extends ProviderV3 {
|
|
34
|
+
/**
|
|
35
|
+
* Creates a Replicate image generation model.
|
|
36
|
+
*/
|
|
37
|
+
image(modelId: ReplicateImageModelId): ReplicateImageModel;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Creates a Replicate image generation model.
|
|
41
|
+
*/
|
|
42
|
+
imageModel(modelId: ReplicateImageModelId): ReplicateImageModel;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* @deprecated Use `embeddingModel` instead.
|
|
46
|
+
*/
|
|
47
|
+
textEmbeddingModel(modelId: string): never;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Create a Replicate provider instance.
|
|
52
|
+
*/
|
|
53
|
+
export function createReplicate(
|
|
54
|
+
options: ReplicateProviderSettings = {},
|
|
55
|
+
): ReplicateProvider {
|
|
56
|
+
const createImageModel = (modelId: ReplicateImageModelId) =>
|
|
57
|
+
new ReplicateImageModel(modelId, {
|
|
58
|
+
provider: 'replicate',
|
|
59
|
+
baseURL: options.baseURL ?? 'https://api.replicate.com/v1',
|
|
60
|
+
headers: withUserAgentSuffix(
|
|
61
|
+
{
|
|
62
|
+
Authorization: `Bearer ${loadApiKey({
|
|
63
|
+
apiKey: options.apiToken,
|
|
64
|
+
environmentVariableName: 'REPLICATE_API_TOKEN',
|
|
65
|
+
description: 'Replicate',
|
|
66
|
+
})}`,
|
|
67
|
+
...options.headers,
|
|
68
|
+
},
|
|
69
|
+
`ai-sdk/replicate/${VERSION}`,
|
|
70
|
+
),
|
|
71
|
+
fetch: options.fetch,
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
const embeddingModel = (modelId: string) => {
|
|
75
|
+
throw new NoSuchModelError({
|
|
76
|
+
modelId,
|
|
77
|
+
modelType: 'embeddingModel',
|
|
78
|
+
});
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
return {
|
|
82
|
+
specificationVersion: 'v3' as const,
|
|
83
|
+
image: createImageModel,
|
|
84
|
+
imageModel: createImageModel,
|
|
85
|
+
languageModel: (modelId: string) => {
|
|
86
|
+
throw new NoSuchModelError({
|
|
87
|
+
modelId,
|
|
88
|
+
modelType: 'languageModel',
|
|
89
|
+
});
|
|
90
|
+
},
|
|
91
|
+
embeddingModel,
|
|
92
|
+
textEmbeddingModel: embeddingModel,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Default Replicate provider instance.
|
|
98
|
+
*/
|
|
99
|
+
export const replicate = createReplicate();
|