modelfusion 0.48.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +186 -183
- package/core/FunctionEvent.d.ts +9 -1
- package/core/FunctionOptions.d.ts +4 -0
- package/core/executeFunctionCall.cjs +85 -0
- package/core/executeFunctionCall.d.ts +10 -0
- package/core/executeFunctionCall.js +81 -0
- package/guard/GuardEvent.d.ts +7 -0
- package/guard/guard.cjs +60 -54
- package/guard/guard.d.ts +2 -1
- package/guard/guard.js +60 -54
- package/guard/index.cjs +1 -0
- package/guard/index.d.ts +1 -0
- package/guard/index.js +1 -0
- package/model-function/AsyncIterableResultPromise.cjs +5 -5
- package/model-function/AsyncIterableResultPromise.d.ts +3 -3
- package/model-function/AsyncIterableResultPromise.js +5 -5
- package/model-function/Model.d.ts +1 -1
- package/model-function/ModelCallEvent.d.ts +5 -7
- package/model-function/embed/EmbeddingEvent.d.ts +2 -2
- package/model-function/embed/embed.cjs +5 -5
- package/model-function/embed/embed.js +5 -5
- package/model-function/{executeCall.cjs → executeStandardCall.cjs} +5 -3
- package/model-function/{executeCall.d.ts → executeStandardCall.d.ts} +1 -1
- package/model-function/{executeCall.js → executeStandardCall.js} +3 -1
- package/model-function/executeStreamCall.cjs +134 -0
- package/model-function/executeStreamCall.d.ts +20 -0
- package/model-function/executeStreamCall.js +130 -0
- package/model-function/generate-image/ImageGenerationEvent.d.ts +2 -2
- package/model-function/generate-image/generateImage.cjs +3 -3
- package/model-function/generate-image/generateImage.js +3 -3
- package/model-function/generate-speech/SpeechGenerationEvent.d.ts +27 -0
- package/model-function/generate-speech/SpeechGenerationModel.d.ts +15 -0
- package/model-function/generate-speech/generateSpeech.cjs +24 -0
- package/model-function/generate-speech/generateSpeech.d.ts +8 -0
- package/model-function/generate-speech/generateSpeech.js +20 -0
- package/model-function/generate-speech/index.cjs +20 -0
- package/model-function/generate-speech/index.d.ts +4 -0
- package/model-function/generate-speech/index.js +4 -0
- package/model-function/generate-speech/streamSpeech.cjs +34 -0
- package/model-function/generate-speech/streamSpeech.d.ts +8 -0
- package/model-function/generate-speech/streamSpeech.js +30 -0
- package/model-function/generate-structure/StructureGenerationEvent.d.ts +2 -2
- package/model-function/generate-structure/StructureStreamingEvent.d.ts +2 -2
- package/model-function/generate-structure/generateStructure.cjs +3 -3
- package/model-function/generate-structure/generateStructure.js +3 -3
- package/model-function/generate-structure/generateStructureOrText.cjs +3 -3
- package/model-function/generate-structure/generateStructureOrText.js +3 -3
- package/model-function/generate-structure/index.cjs +27 -0
- package/model-function/generate-structure/index.d.ts +11 -0
- package/model-function/generate-structure/index.js +11 -0
- package/model-function/generate-structure/streamStructure.cjs +29 -137
- package/model-function/generate-structure/streamStructure.js +28 -136
- package/model-function/generate-text/TextGenerationEvent.d.ts +8 -2
- package/model-function/generate-text/generateText.cjs +4 -4
- package/model-function/generate-text/generateText.d.ts +1 -1
- package/model-function/generate-text/generateText.js +4 -4
- package/model-function/generate-text/index.cjs +0 -1
- package/model-function/generate-text/index.d.ts +0 -1
- package/model-function/generate-text/index.js +0 -1
- package/model-function/generate-text/streamText.cjs +22 -129
- package/model-function/generate-text/streamText.js +21 -128
- package/model-function/generate-text/trimChatPrompt.cjs +1 -1
- package/model-function/generate-text/trimChatPrompt.d.ts +1 -1
- package/model-function/generate-text/trimChatPrompt.js +1 -1
- package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.d.ts +2 -2
- package/model-function/{transcribe-speech/transcribe.cjs → generate-transcription/generateTranscription.cjs} +7 -7
- package/model-function/{transcribe-speech/transcribe.d.ts → generate-transcription/generateTranscription.d.ts} +2 -2
- package/model-function/{transcribe-speech/transcribe.js → generate-transcription/generateTranscription.js} +5 -5
- package/model-function/index.cjs +5 -20
- package/model-function/index.d.ts +5 -20
- package/model-function/index.js +5 -20
- package/model-provider/elevenlabs/{ElevenLabsSpeechSynthesisModel.cjs → ElevenLabsSpeechModel.cjs} +6 -6
- package/model-provider/elevenlabs/{ElevenLabsSpeechSynthesisModel.d.ts → ElevenLabsSpeechModel.d.ts} +8 -8
- package/model-provider/elevenlabs/{ElevenLabsSpeechSynthesisModel.js → ElevenLabsSpeechModel.js} +4 -4
- package/model-provider/elevenlabs/index.cjs +1 -1
- package/model-provider/elevenlabs/index.d.ts +1 -1
- package/model-provider/elevenlabs/index.js +1 -1
- package/model-provider/huggingface/HuggingFaceImageDescriptionModel.cjs +21 -2
- package/model-provider/huggingface/HuggingFaceImageDescriptionModel.d.ts +11 -6
- package/model-provider/huggingface/HuggingFaceImageDescriptionModel.js +21 -2
- package/model-provider/lmnt/{LmntSpeechSynthesisModel.cjs → LmntSpeechModel.cjs} +5 -5
- package/model-provider/lmnt/LmntSpeechModel.d.ts +26 -0
- package/model-provider/lmnt/{LmntSpeechSynthesisModel.js → LmntSpeechModel.js} +3 -3
- package/model-provider/lmnt/index.cjs +1 -1
- package/model-provider/lmnt/index.d.ts +1 -1
- package/model-provider/lmnt/index.js +1 -1
- package/model-provider/openai/{OpenAITextGenerationModel.cjs → OpenAICompletionModel.cjs} +17 -17
- package/model-provider/openai/{OpenAITextGenerationModel.d.ts → OpenAICompletionModel.d.ts} +29 -29
- package/model-provider/openai/{OpenAITextGenerationModel.js → OpenAICompletionModel.js} +12 -12
- package/model-provider/openai/OpenAICostCalculator.cjs +8 -8
- package/model-provider/openai/OpenAICostCalculator.js +8 -8
- package/model-provider/openai/OpenAITextEmbeddingModel.d.ts +3 -3
- package/model-provider/openai/OpenAITranscriptionModel.d.ts +1 -1
- package/model-provider/openai/TikTokenTokenizer.d.ts +2 -2
- package/model-provider/openai/chat/OpenAIChatModel.d.ts +7 -7
- package/model-provider/openai/index.cjs +1 -1
- package/model-provider/openai/index.d.ts +1 -1
- package/model-provider/openai/index.js +1 -1
- package/package.json +1 -1
- package/retriever/retrieve.cjs +7 -75
- package/retriever/retrieve.js +7 -75
- package/tool/UseToolEvent.d.ts +7 -0
- package/tool/UseToolOrGenerateTextEvent.d.ts +7 -0
- package/tool/executeTool.cjs +2 -0
- package/tool/executeTool.js +2 -0
- package/tool/index.cjs +2 -0
- package/tool/index.d.ts +2 -0
- package/tool/index.js +2 -0
- package/tool/useTool.cjs +18 -10
- package/tool/useTool.js +18 -10
- package/tool/useToolOrGenerateText.cjs +34 -26
- package/tool/useToolOrGenerateText.js +34 -26
- package/vector-index/UpsertIntoVectorIndexEvent.cjs +2 -0
- package/vector-index/UpsertIntoVectorIndexEvent.d.ts +9 -0
- package/vector-index/UpsertIntoVectorIndexEvent.js +1 -0
- package/vector-index/VectorIndexRetriever.cjs +1 -4
- package/vector-index/VectorIndexRetriever.js +1 -4
- package/vector-index/index.cjs +1 -0
- package/vector-index/index.d.ts +1 -0
- package/vector-index/index.js +1 -0
- package/vector-index/upsertIntoVectorIndex.cjs +16 -7
- package/vector-index/upsertIntoVectorIndex.js +16 -7
- package/model-function/describe-image/ImageDescriptionEvent.d.ts +0 -18
- package/model-function/describe-image/ImageDescriptionModel.d.ts +0 -10
- package/model-function/describe-image/describeImage.cjs +0 -26
- package/model-function/describe-image/describeImage.d.ts +0 -9
- package/model-function/describe-image/describeImage.js +0 -22
- package/model-function/generate-text/TextStreamingEvent.d.ts +0 -7
- package/model-function/synthesize-speech/SpeechSynthesisEvent.d.ts +0 -21
- package/model-function/synthesize-speech/SpeechSynthesisModel.d.ts +0 -15
- package/model-function/synthesize-speech/synthesizeSpeech.cjs +0 -67
- package/model-function/synthesize-speech/synthesizeSpeech.d.ts +0 -14
- package/model-function/synthesize-speech/synthesizeSpeech.js +0 -63
- package/model-provider/lmnt/LmntSpeechSynthesisModel.d.ts +0 -26
- /package/{model-function/describe-image/ImageDescriptionEvent.cjs → guard/GuardEvent.cjs} +0 -0
- /package/{model-function/describe-image/ImageDescriptionEvent.js → guard/GuardEvent.js} +0 -0
- /package/model-function/{describe-image/ImageDescriptionModel.cjs → generate-speech/SpeechGenerationEvent.cjs} +0 -0
- /package/model-function/{describe-image/ImageDescriptionModel.js → generate-speech/SpeechGenerationEvent.js} +0 -0
- /package/model-function/{generate-text/TextStreamingEvent.cjs → generate-speech/SpeechGenerationModel.cjs} +0 -0
- /package/model-function/{generate-text/TextStreamingEvent.js → generate-speech/SpeechGenerationModel.js} +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.cjs +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.js +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.cjs +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.d.ts +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.js +0 -0
- /package/{model-function/synthesize-speech/SpeechSynthesisEvent.cjs → tool/UseToolEvent.cjs} +0 -0
- /package/{model-function/synthesize-speech/SpeechSynthesisEvent.js → tool/UseToolEvent.js} +0 -0
- /package/{model-function/synthesize-speech/SpeechSynthesisModel.cjs → tool/UseToolOrGenerateTextEvent.cjs} +0 -0
- /package/{model-function/synthesize-speech/SpeechSynthesisModel.js → tool/UseToolOrGenerateTextEvent.js} +0 -0
package/README.md
CHANGED
@@ -15,13 +15,14 @@
|
|
15
15
|
|
16
16
|
## Introduction
|
17
17
|
|
18
|
-
ModelFusion is a library for building AI
|
18
|
+
ModelFusion is a library for building AI applications, chatbots, and agents. Here are the main features:
|
19
19
|
|
20
|
-
- **Multimodal
|
21
|
-
- **
|
22
|
-
- **
|
23
|
-
- **
|
24
|
-
- **
|
20
|
+
- **Multimodal**: ModelFusion supports a wide range of models including text generation, image generation, text-to-speech, speech-to-text, and embedding models.
|
21
|
+
- **Streaming**: ModelFusion supports streaming for many generation models, e.g. text streaming, structure streaming, and full duplex speech streaming.
|
22
|
+
- **Utility functions**: ModelFusion provides functionality for tools and tool usage, vector indices, and guards functions.
|
23
|
+
- **Type inference and validation**: ModelFusion infers TypeScript types wherever possible and to validates model responses.
|
24
|
+
- **Observability and logging**: ModelFusion provides an observer framework and out-of-the-box logging support.
|
25
|
+
- **Resilience and Robustness**: ModelFusion ensures seamless operation through automatic retries, throttling, and error handling mechanisms.
|
25
26
|
|
26
27
|
## Quick Install
|
27
28
|
|
@@ -35,17 +36,17 @@ Or use a template: [ModelFusion terminal app starter](https://github.com/lgramme
|
|
35
36
|
|
36
37
|
You can provide API keys for the different [integrations](https://modelfusion.dev/integration/model-provider/) using environment variables (e.g., `OPENAI_API_KEY`) or pass them into the model constructors as options.
|
37
38
|
|
38
|
-
### [Generate
|
39
|
+
### [Generate Text](https://modelfusion.dev/guide/function/generate-text)
|
39
40
|
|
40
41
|
Generate text using a language model and a prompt.
|
41
42
|
You can stream the text if it is supported by the model.
|
42
|
-
You can use [prompt formats](https://modelfusion.dev/guide/function/generate-text
|
43
|
+
You can use [prompt formats](https://modelfusion.dev/guide/function/generate-text#prompt-format) to change the prompt format of a model.
|
43
44
|
|
44
45
|
#### generateText
|
45
46
|
|
46
47
|
```ts
|
47
48
|
const text = await generateText(
|
48
|
-
new
|
49
|
+
new OpenAICompletionModel({
|
49
50
|
model: "gpt-3.5-turbo-instruct",
|
50
51
|
}),
|
51
52
|
"Write a short story about a robot learning to love:\n\n"
|
@@ -58,7 +59,7 @@ Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai),
|
|
58
59
|
|
59
60
|
```ts
|
60
61
|
const textStream = await streamText(
|
61
|
-
new
|
62
|
+
new OpenAICompletionModel({
|
62
63
|
model: "gpt-3.5-turbo-instruct",
|
63
64
|
}),
|
64
65
|
"Write a short story about a robot learning to love:\n\n"
|
@@ -71,7 +72,85 @@ for await (const textFragment of textStream) {
|
|
71
72
|
|
72
73
|
Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp)
|
73
74
|
|
74
|
-
### [Generate
|
75
|
+
### [Generate Image](https://modelfusion.dev/guide/function/generate-image)
|
76
|
+
|
77
|
+
Generate an image from a prompt.
|
78
|
+
|
79
|
+
```ts
|
80
|
+
const image = await generateImage(
|
81
|
+
new OpenAIImageGenerationModel({ size: "512x512" }),
|
82
|
+
"the wicked witch of the west in the style of early 19th century painting"
|
83
|
+
);
|
84
|
+
```
|
85
|
+
|
86
|
+
Providers: [OpenAI (Dall·E)](https://modelfusion.dev/integration/model-provider/openai), [Stability AI](https://modelfusion.dev/integration/model-provider/stability), [Automatic1111](https://modelfusion.dev/integration/model-provider/automatic1111)
|
87
|
+
|
88
|
+
### [Generate Speech](https://modelfusion.dev/guide/function/generate-speech)
|
89
|
+
|
90
|
+
Synthesize speech (audio) from text. Also called TTS (text-to-speech).
|
91
|
+
|
92
|
+
#### generateSpeech
|
93
|
+
|
94
|
+
`generateSpeech` synthesizes speech from text.
|
95
|
+
|
96
|
+
```ts
|
97
|
+
// `speech` is a Buffer with MP3 audio data
|
98
|
+
const speech = await generateSpeech(
|
99
|
+
new LmntSpeechModel({
|
100
|
+
voice: "034b632b-df71-46c8-b440-86a42ffc3cf3", // Henry
|
101
|
+
}),
|
102
|
+
"Good evening, ladies and gentlemen! Exciting news on the airwaves tonight " +
|
103
|
+
"as The Rolling Stones unveil 'Hackney Diamonds,' their first collection of " +
|
104
|
+
"fresh tunes in nearly twenty years, featuring the illustrious Lady Gaga, the " +
|
105
|
+
"magical Stevie Wonder, and the final beats from the late Charlie Watts."
|
106
|
+
);
|
107
|
+
```
|
108
|
+
|
109
|
+
Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs), [LMNT](https://modelfusion.dev/integration/model-provider/lmnt)
|
110
|
+
|
111
|
+
#### streamSpeech
|
112
|
+
|
113
|
+
`generateSpeech` generates a stream of speech chunks from text or from a text stream. Depending on the model, this can be fully duplex.
|
114
|
+
|
115
|
+
```ts
|
116
|
+
const textStream = await streamText(/* ... */);
|
117
|
+
|
118
|
+
const speechStream = await streamSpeech(
|
119
|
+
new ElevenLabsSpeechModel({
|
120
|
+
voice: "pNInz6obpgDQGcFmaJgB", // Adam
|
121
|
+
model: "eleven_monolingual_v1",
|
122
|
+
voiceSettings: { stability: 1, similarityBoost: 0.35 },
|
123
|
+
generationConfig: {
|
124
|
+
chunkLengthSchedule: [50, 90, 120, 150, 200],
|
125
|
+
},
|
126
|
+
}),
|
127
|
+
textStream
|
128
|
+
);
|
129
|
+
|
130
|
+
for await (const part of speechStream) {
|
131
|
+
// each part is a Buffer with MP3 audio data
|
132
|
+
}
|
133
|
+
```
|
134
|
+
|
135
|
+
Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs)
|
136
|
+
|
137
|
+
### [Generate Transcription](https://modelfusion.dev/guide/function/generate-transcription)
|
138
|
+
|
139
|
+
Transcribe speech (audio) data into text. Also called speech-to-text (STT).
|
140
|
+
|
141
|
+
```ts
|
142
|
+
const transcription = await generateTranscription(
|
143
|
+
new OpenAITranscriptionModel({ model: "whisper-1" }),
|
144
|
+
{
|
145
|
+
type: "mp3",
|
146
|
+
data: await fs.promises.readFile("data/test.mp3"),
|
147
|
+
}
|
148
|
+
);
|
149
|
+
```
|
150
|
+
|
151
|
+
Providers: [OpenAI (Whisper)](https://modelfusion.dev/integration/model-provider/openai)
|
152
|
+
|
153
|
+
### [Generate Structure](https://modelfusion.dev/guide/function/generate-structure#generatestructure)
|
75
154
|
|
76
155
|
Generate typed objects using a language model and a schema.
|
77
156
|
|
@@ -189,161 +268,6 @@ const { structure, value, text } = await generateStructureOrText(
|
|
189
268
|
|
190
269
|
Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai)
|
191
270
|
|
192
|
-
### [Tools](https://modelfusion.dev/guide/tools)
|
193
|
-
|
194
|
-
Tools are functions that can be executed by an AI model. They are useful for building chatbots and agents.
|
195
|
-
|
196
|
-
Predefined tools: [SerpAPI](https://modelfusion.dev/integration/tool/serpapi), [Google Custom Search](https://modelfusion.dev/integration/tool/google-custom-search)
|
197
|
-
|
198
|
-
#### Create Tool
|
199
|
-
|
200
|
-
A tool is a function with a name, a description, and a schema for the input parameters.
|
201
|
-
|
202
|
-
```ts
|
203
|
-
const calculator = new Tool({
|
204
|
-
name: "calculator",
|
205
|
-
description: "Execute a calculation",
|
206
|
-
|
207
|
-
inputSchema: new ZodSchema(
|
208
|
-
z.object({
|
209
|
-
a: z.number().describe("The first number."),
|
210
|
-
b: z.number().describe("The second number."),
|
211
|
-
operator: z
|
212
|
-
.enum(["+", "-", "*", "/"])
|
213
|
-
.describe("The operator (+, -, *, /)."),
|
214
|
-
})
|
215
|
-
),
|
216
|
-
|
217
|
-
execute: async ({ a, b, operator }) => {
|
218
|
-
switch (operator) {
|
219
|
-
case "+":
|
220
|
-
return a + b;
|
221
|
-
case "-":
|
222
|
-
return a - b;
|
223
|
-
case "*":
|
224
|
-
return a * b;
|
225
|
-
case "/":
|
226
|
-
return a / b;
|
227
|
-
default:
|
228
|
-
throw new Error(`Unknown operator: ${operator}`);
|
229
|
-
}
|
230
|
-
},
|
231
|
-
});
|
232
|
-
```
|
233
|
-
|
234
|
-
#### useTool
|
235
|
-
|
236
|
-
The model determines the parameters for the tool from the prompt and then executes it.
|
237
|
-
|
238
|
-
```ts
|
239
|
-
const { tool, parameters, result } = await useTool(
|
240
|
-
new OpenAIChatModel({ model: "gpt-3.5-turbo" }),
|
241
|
-
calculator,
|
242
|
-
[OpenAIChatMessage.user("What's fourteen times twelve?")]
|
243
|
-
);
|
244
|
-
```
|
245
|
-
|
246
|
-
#### useToolOrGenerateText
|
247
|
-
|
248
|
-
The model determines which tool to use and its parameters from the prompt and then executes it.
|
249
|
-
Text is generated as a fallback.
|
250
|
-
|
251
|
-
```ts
|
252
|
-
const { tool, parameters, result, text } = await useToolOrGenerateText(
|
253
|
-
new OpenAIChatModel({ model: "gpt-3.5-turbo" }),
|
254
|
-
[calculator /* and other tools... */],
|
255
|
-
[OpenAIChatMessage.user("What's fourteen times twelve?")]
|
256
|
-
);
|
257
|
-
```
|
258
|
-
|
259
|
-
### [Transcribe Speech](https://modelfusion.dev/guide/function/transcribe-speech)
|
260
|
-
|
261
|
-
Turn speech (audio) into text.
|
262
|
-
|
263
|
-
```ts
|
264
|
-
const transcription = await transcribe(
|
265
|
-
new OpenAITranscriptionModel({ model: "whisper-1" }),
|
266
|
-
{
|
267
|
-
type: "mp3",
|
268
|
-
data: await fs.promises.readFile("data/test.mp3"),
|
269
|
-
}
|
270
|
-
);
|
271
|
-
```
|
272
|
-
|
273
|
-
Providers: [OpenAI (Whisper)](https://modelfusion.dev/integration/model-provider/openai)
|
274
|
-
|
275
|
-
### [Synthesize Speech](https://modelfusion.dev/guide/function/synthesize-speech)
|
276
|
-
|
277
|
-
Generate speech (audio) from text. Also called TTS (text-to-speech).
|
278
|
-
|
279
|
-
Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs), [LMNT](https://modelfusion.dev/integration/model-provider/lmnt)
|
280
|
-
|
281
|
-
#### Standard mode
|
282
|
-
|
283
|
-
```ts
|
284
|
-
// `speech` is a Buffer with MP3 audio data
|
285
|
-
const speech = await synthesizeSpeech(
|
286
|
-
new LmntSpeechSynthesisModel({
|
287
|
-
voice: "034b632b-df71-46c8-b440-86a42ffc3cf3", // Henry
|
288
|
-
}),
|
289
|
-
"Good evening, ladies and gentlemen! Exciting news on the airwaves tonight " +
|
290
|
-
"as The Rolling Stones unveil 'Hackney Diamonds,' their first collection of " +
|
291
|
-
"fresh tunes in nearly twenty years, featuring the illustrious Lady Gaga, the " +
|
292
|
-
"magical Stevie Wonder, and the final beats from the late Charlie Watts."
|
293
|
-
);
|
294
|
-
```
|
295
|
-
|
296
|
-
#### Duplex streaming mode
|
297
|
-
|
298
|
-
```ts
|
299
|
-
const textStream = await streamText(/* ... */);
|
300
|
-
|
301
|
-
const speechStream = await synthesizeSpeech(
|
302
|
-
new ElevenLabsSpeechSynthesisModel({
|
303
|
-
voice: "pNInz6obpgDQGcFmaJgB", // Adam
|
304
|
-
model: "eleven_monolingual_v1",
|
305
|
-
voiceSettings: { stability: 1, similarityBoost: 0.35 },
|
306
|
-
generationConfig: {
|
307
|
-
chunkLengthSchedule: [50, 90, 120, 150, 200],
|
308
|
-
},
|
309
|
-
}),
|
310
|
-
textStream,
|
311
|
-
{ mode: "stream-duplex" }
|
312
|
-
);
|
313
|
-
|
314
|
-
for await (const part of speechStream) {
|
315
|
-
// each part is a Buffer with MP3 audio data
|
316
|
-
}
|
317
|
-
```
|
318
|
-
|
319
|
-
### [Describe Image](https://modelfusion.dev/guide/function/describe-image)
|
320
|
-
|
321
|
-
Describe an image as text, e.g. for image captioning or OCR.
|
322
|
-
|
323
|
-
```ts
|
324
|
-
const text = await describeImage(
|
325
|
-
new HuggingFaceImageDescriptionModel({
|
326
|
-
model: "nlpconnect/vit-gpt2-image-captioning",
|
327
|
-
}),
|
328
|
-
data // buffer with image data
|
329
|
-
);
|
330
|
-
```
|
331
|
-
|
332
|
-
Providers: [HuggingFace](/integration/model-provider/huggingface)
|
333
|
-
|
334
|
-
### [Generate Image](https://modelfusion.dev/guide/function/generate-image)
|
335
|
-
|
336
|
-
Generate an image from a prompt.
|
337
|
-
|
338
|
-
```ts
|
339
|
-
const image = await generateImage(
|
340
|
-
new OpenAIImageGenerationModel({ size: "512x512" }),
|
341
|
-
"the wicked witch of the west in the style of early 19th century painting"
|
342
|
-
);
|
343
|
-
```
|
344
|
-
|
345
|
-
Providers: [OpenAI (Dall·E)](https://modelfusion.dev/integration/model-provider/openai), [Stability AI](https://modelfusion.dev/integration/model-provider/stability), [Automatic1111](https://modelfusion.dev/integration/model-provider/automatic1111)
|
346
|
-
|
347
271
|
### [Embed Value](https://modelfusion.dev/guide/function/embed)
|
348
272
|
|
349
273
|
Create embeddings for text and other values. Embeddings are vectors that represent the essence of the values in the context of the model.
|
@@ -393,7 +317,7 @@ Guard functions can be used to implement retry on error, redacting and changing
|
|
393
317
|
|
394
318
|
```ts
|
395
319
|
const result = await guard(
|
396
|
-
(input) =>
|
320
|
+
(input, options) =>
|
397
321
|
generateStructure(
|
398
322
|
new OpenAIChatModel({
|
399
323
|
// ...
|
@@ -401,7 +325,8 @@ const result = await guard(
|
|
401
325
|
new ZodStructureDefinition({
|
402
326
|
// ...
|
403
327
|
}),
|
404
|
-
input
|
328
|
+
input,
|
329
|
+
options
|
405
330
|
),
|
406
331
|
[
|
407
332
|
// ...
|
@@ -420,7 +345,74 @@ const result = await guard(
|
|
420
345
|
);
|
421
346
|
```
|
422
347
|
|
423
|
-
### [
|
348
|
+
### [Tools](https://modelfusion.dev/guide/tools)
|
349
|
+
|
350
|
+
Tools are functions that can be executed by an AI model. They are useful for building chatbots and agents.
|
351
|
+
|
352
|
+
Predefined tools: [SerpAPI](https://modelfusion.dev/integration/tool/serpapi), [Google Custom Search](https://modelfusion.dev/integration/tool/google-custom-search)
|
353
|
+
|
354
|
+
#### Create Tool
|
355
|
+
|
356
|
+
A tool is a function with a name, a description, and a schema for the input parameters.
|
357
|
+
|
358
|
+
```ts
|
359
|
+
const calculator = new Tool({
|
360
|
+
name: "calculator",
|
361
|
+
description: "Execute a calculation",
|
362
|
+
|
363
|
+
inputSchema: new ZodSchema(
|
364
|
+
z.object({
|
365
|
+
a: z.number().describe("The first number."),
|
366
|
+
b: z.number().describe("The second number."),
|
367
|
+
operator: z
|
368
|
+
.enum(["+", "-", "*", "/"])
|
369
|
+
.describe("The operator (+, -, *, /)."),
|
370
|
+
})
|
371
|
+
),
|
372
|
+
|
373
|
+
execute: async ({ a, b, operator }) => {
|
374
|
+
switch (operator) {
|
375
|
+
case "+":
|
376
|
+
return a + b;
|
377
|
+
case "-":
|
378
|
+
return a - b;
|
379
|
+
case "*":
|
380
|
+
return a * b;
|
381
|
+
case "/":
|
382
|
+
return a / b;
|
383
|
+
default:
|
384
|
+
throw new Error(`Unknown operator: ${operator}`);
|
385
|
+
}
|
386
|
+
},
|
387
|
+
});
|
388
|
+
```
|
389
|
+
|
390
|
+
#### useTool
|
391
|
+
|
392
|
+
The model determines the parameters for the tool from the prompt and then executes it.
|
393
|
+
|
394
|
+
```ts
|
395
|
+
const { tool, parameters, result } = await useTool(
|
396
|
+
new OpenAIChatModel({ model: "gpt-3.5-turbo" }),
|
397
|
+
calculator,
|
398
|
+
[OpenAIChatMessage.user("What's fourteen times twelve?")]
|
399
|
+
);
|
400
|
+
```
|
401
|
+
|
402
|
+
#### useToolOrGenerateText
|
403
|
+
|
404
|
+
The model determines which tool to use and its parameters from the prompt and then executes it.
|
405
|
+
Text is generated as a fallback.
|
406
|
+
|
407
|
+
```ts
|
408
|
+
const { tool, parameters, result, text } = await useToolOrGenerateText(
|
409
|
+
new OpenAIChatModel({ model: "gpt-3.5-turbo" }),
|
410
|
+
[calculator /* and other tools... */],
|
411
|
+
[OpenAIChatMessage.user("What's fourteen times twelve?")]
|
412
|
+
);
|
413
|
+
```
|
414
|
+
|
415
|
+
### [Vector Indices](https://modelfusion.dev/guide/vector-index)
|
424
416
|
|
425
417
|
```ts
|
426
418
|
const texts = [
|
@@ -460,7 +452,7 @@ Available Vector Stores: [Memory](https://modelfusion.dev/integration/vector-ind
|
|
460
452
|
|
461
453
|
Prompt formats let you use higher level prompt structures (such as instruction or chat prompts) for different models.
|
462
454
|
|
463
|
-
#### [Text Generation Prompt Formats](https://modelfusion.dev/guide/function/generate-text
|
455
|
+
#### [Text Generation Prompt Formats](https://modelfusion.dev/guide/function/generate-text#prompt-format)
|
464
456
|
|
465
457
|
```ts
|
466
458
|
const text = await generateText(
|
@@ -525,7 +517,7 @@ ModelFusion model functions return rich results that include the original respon
|
|
525
517
|
```ts
|
526
518
|
// access the full response (needs to be typed) and the metadata:
|
527
519
|
const { value, response, metadata } = await generateText(
|
528
|
-
new
|
520
|
+
new OpenAICompletionModel({
|
529
521
|
model: "gpt-3.5-turbo-instruct",
|
530
522
|
maxCompletionTokens: 1000,
|
531
523
|
n: 2, // generate 2 completions
|
@@ -536,31 +528,34 @@ const { value, response, metadata } = await generateText(
|
|
536
528
|
console.log(metadata);
|
537
529
|
|
538
530
|
// cast to the response type:
|
539
|
-
for (const choice of (response as
|
531
|
+
for (const choice of (response as OpenAICompletionResponse).choices) {
|
540
532
|
console.log(choice.text);
|
541
533
|
}
|
542
534
|
```
|
543
535
|
|
544
|
-
### Observability
|
536
|
+
### Logging and Observability
|
545
537
|
|
546
|
-
|
538
|
+
ModelFusion provides an [observer framework](https://modelfusion.dev/guide/util/observer) and [out-of-the-box logging support](https://modelfusion.dev/guide/util/logging). You can easily trace runs and call hierarchies, and you can add your own observers.
|
539
|
+
|
540
|
+
#### Global Logging Example
|
541
|
+
|
542
|
+
```ts
|
543
|
+
setGlobalFunctionLogging("detailed-object"); // log full events
|
544
|
+
```
|
547
545
|
|
548
546
|
## Documentation
|
549
547
|
|
550
548
|
### [Guide](https://modelfusion.dev/guide)
|
551
549
|
|
552
550
|
- [Model Functions](https://modelfusion.dev/guide/function/)
|
553
|
-
- [Generate
|
554
|
-
|
555
|
-
- [Generate
|
551
|
+
- [Generate text](https://modelfusion.dev/guide/function/generate-text)
|
552
|
+
- [Generate image](https://modelfusion.dev/guide/function/generate-image)
|
553
|
+
- [Generate speech](https://modelfusion.dev/guide/function/generate-speech)
|
554
|
+
- [Generate transcription](https://modelfusion.dev/guide/function/generation-transcription)
|
555
|
+
- [Generate structure](https://modelfusion.dev/guide/function/generate-structure)
|
556
556
|
- [Generate structure or text](https://modelfusion.dev/guide/function/generate-structure-or-text)
|
557
557
|
- [Tokenize Text](https://modelfusion.dev/guide/function/tokenize-text)
|
558
558
|
- [Embed Value](https://modelfusion.dev/guide/function/embed)
|
559
|
-
- [Transcribe Speech](https://modelfusion.dev/guide/function/transcribe-speech)
|
560
|
-
- [Synthesize Speech](https://modelfusion.dev/guide/function/synthesize-speech)
|
561
|
-
- [Describe Image](https://modelfusion.dev/guide/function/describe-image)
|
562
|
-
- [Generate Image](https://modelfusion.dev/guide/function/generate-image)
|
563
|
-
- [Prompt Format](https://modelfusion.dev/guide/function/generate-image/prompt-format)
|
564
559
|
- [Guards](https://modelfusion.dev/guide/guard)
|
565
560
|
- [Tools](https://modelfusion.dev/guide/tools)
|
566
561
|
- [Vector Indices](https://modelfusion.dev/guide/vector-index)
|
@@ -628,7 +623,7 @@ Create an 19th century painting image for your input.
|
|
628
623
|
|
629
624
|
Record audio with push-to-talk and transcribe it using Whisper, implemented as a Next.js app. The app shows a list of the transcriptions.
|
630
625
|
|
631
|
-
### [
|
626
|
+
### [Speech Streaming (Vite(React) + Fastify))](https://github.com/lgrammel/modelfusion/tree/main/examples/speech-streaming-vite-react-fastify)
|
632
627
|
|
633
628
|
> _Speech Streaming_, _OpenAI_, _Elevenlabs_ _streaming_, _Vite_, _Fastify_
|
634
629
|
|
@@ -669,3 +664,11 @@ Generate text on a Cloudflare Worker using ModelFusion and OpenAI.
|
|
669
664
|
### [Contributing Guide](https://github.com/lgrammel/modelfusion/blob/main/CONTRIBUTING.md)
|
670
665
|
|
671
666
|
Read the [ModelFusion contributing guide](https://github.com/lgrammel/modelfusion/blob/main/CONTRIBUTING.md) to learn about the development process, how to propose bugfixes and improvements, and how to build and test your changes.
|
667
|
+
|
668
|
+
```
|
669
|
+
|
670
|
+
```
|
671
|
+
|
672
|
+
```
|
673
|
+
|
674
|
+
```
|
package/core/FunctionEvent.d.ts
CHANGED
@@ -1,11 +1,19 @@
|
|
1
|
+
import { GuardFinishedEvent, GuardStartedEvent } from "../guard/GuardEvent.js";
|
1
2
|
import { ModelCallFinishedEvent, ModelCallStartedEvent } from "../model-function/ModelCallEvent.js";
|
2
3
|
import { RetrieveFinishedEvent, RetrieveStartedEvent } from "../retriever/RetrieveEvent.js";
|
3
4
|
import { ExecuteToolFinishedEvent, ExecuteToolStartedEvent } from "../tool/ExecuteToolEvent.js";
|
5
|
+
import { UseToolFinishedEvent, UseToolStartedEvent } from "../tool/UseToolEvent.js";
|
6
|
+
import { UseToolOrGenerateTextFinishedEvent, UseToolOrGenerateTextStartedEvent } from "../tool/UseToolOrGenerateTextEvent.js";
|
7
|
+
import { UpsertIntoVectorIndexFinishedEvent, UpsertIntoVectorIndexStartedEvent } from "../vector-index/UpsertIntoVectorIndexEvent.js";
|
4
8
|
export interface BaseFunctionEvent {
|
5
9
|
/**
|
6
10
|
* Unique identifier for the function call.
|
7
11
|
*/
|
8
12
|
callId: string | undefined;
|
13
|
+
/**
|
14
|
+
* Unique identifier of the call id of the parent function.
|
15
|
+
*/
|
16
|
+
parentCallId?: string | undefined;
|
9
17
|
/**
|
10
18
|
* Optional unique identifier for the function.
|
11
19
|
*/
|
@@ -73,4 +81,4 @@ export interface BaseFunctionFinishedEvent extends BaseFunctionEvent {
|
|
73
81
|
*/
|
74
82
|
result: BaseFunctionFinishedEventResult;
|
75
83
|
}
|
76
|
-
export type FunctionEvent = ModelCallStartedEvent |
|
84
|
+
export type FunctionEvent = ExecuteToolStartedEvent | ExecuteToolFinishedEvent | GuardStartedEvent | GuardFinishedEvent | ModelCallStartedEvent | ModelCallFinishedEvent | RetrieveStartedEvent | RetrieveFinishedEvent | UpsertIntoVectorIndexStartedEvent | UpsertIntoVectorIndexFinishedEvent | UseToolStartedEvent | UseToolFinishedEvent | UseToolOrGenerateTextStartedEvent | UseToolOrGenerateTextFinishedEvent;
|
@@ -21,6 +21,10 @@ export type FunctionOptions = {
|
|
21
21
|
* Optional run as part of which this function is called.
|
22
22
|
*/
|
23
23
|
run?: Run;
|
24
|
+
/**
|
25
|
+
* Unique identifier of the call id of the parent function.
|
26
|
+
*/
|
27
|
+
parentCallId?: string | undefined;
|
24
28
|
};
|
25
29
|
/**
|
26
30
|
* The logging to use for the function. Logs are sent to the console.
|
@@ -0,0 +1,85 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.executeFunctionCall = void 0;
|
4
|
+
const nanoid_1 = require("nanoid");
|
5
|
+
const FunctionEventSource_js_1 = require("./FunctionEventSource.cjs");
|
6
|
+
const GlobalFunctionLogging_js_1 = require("./GlobalFunctionLogging.cjs");
|
7
|
+
const GlobalFunctionObservers_js_1 = require("./GlobalFunctionObservers.cjs");
|
8
|
+
const AbortError_js_1 = require("./api/AbortError.cjs");
|
9
|
+
const getFunctionCallLogger_js_1 = require("./getFunctionCallLogger.cjs");
|
10
|
+
const getRun_js_1 = require("./getRun.cjs");
|
11
|
+
const DurationMeasurement_js_1 = require("../util/DurationMeasurement.cjs");
|
12
|
+
const runSafe_js_1 = require("../util/runSafe.cjs");
|
13
|
+
async function executeFunctionCall({ options, input, functionType, execute, inputPropertyName = "input", outputPropertyName = "value", }) {
|
14
|
+
const run = await (0, getRun_js_1.getRun)(options?.run);
|
15
|
+
const eventSource = new FunctionEventSource_js_1.FunctionEventSource({
|
16
|
+
observers: [
|
17
|
+
...(0, getFunctionCallLogger_js_1.getFunctionCallLogger)(options?.logging ?? (0, GlobalFunctionLogging_js_1.getGlobalFunctionLogging)()),
|
18
|
+
...(0, GlobalFunctionObservers_js_1.getGlobalFunctionObservers)(),
|
19
|
+
...(run?.functionObserver != null ? [run.functionObserver] : []),
|
20
|
+
...(options?.observers ?? []),
|
21
|
+
],
|
22
|
+
errorHandler: run?.errorHandler,
|
23
|
+
});
|
24
|
+
const durationMeasurement = (0, DurationMeasurement_js_1.startDurationMeasurement)();
|
25
|
+
const startMetadata = {
|
26
|
+
functionType,
|
27
|
+
callId: `call-${(0, nanoid_1.nanoid)()}`,
|
28
|
+
parentCallId: options?.parentCallId,
|
29
|
+
runId: run?.runId,
|
30
|
+
sessionId: run?.sessionId,
|
31
|
+
userId: run?.userId,
|
32
|
+
functionId: options?.functionId,
|
33
|
+
[inputPropertyName]: input,
|
34
|
+
timestamp: durationMeasurement.startDate,
|
35
|
+
startTimestamp: durationMeasurement.startDate,
|
36
|
+
};
|
37
|
+
eventSource.notify({
|
38
|
+
eventType: "started",
|
39
|
+
...startMetadata,
|
40
|
+
});
|
41
|
+
const result = await (0, runSafe_js_1.runSafe)(() => execute({
|
42
|
+
functionId: options?.functionId,
|
43
|
+
logging: options?.logging,
|
44
|
+
observers: options?.observers,
|
45
|
+
run,
|
46
|
+
parentCallId: startMetadata.callId,
|
47
|
+
}));
|
48
|
+
const finishMetadata = {
|
49
|
+
eventType: "finished",
|
50
|
+
...startMetadata,
|
51
|
+
finishTimestamp: new Date(),
|
52
|
+
durationInMs: durationMeasurement.durationInMs,
|
53
|
+
};
|
54
|
+
if (!result.ok) {
|
55
|
+
if (result.isAborted) {
|
56
|
+
eventSource.notify({
|
57
|
+
...finishMetadata,
|
58
|
+
eventType: "finished",
|
59
|
+
result: {
|
60
|
+
status: "abort",
|
61
|
+
},
|
62
|
+
});
|
63
|
+
throw new AbortError_js_1.AbortError();
|
64
|
+
}
|
65
|
+
eventSource.notify({
|
66
|
+
...finishMetadata,
|
67
|
+
eventType: "finished",
|
68
|
+
result: {
|
69
|
+
status: "error",
|
70
|
+
error: result.error,
|
71
|
+
},
|
72
|
+
});
|
73
|
+
throw result.error;
|
74
|
+
}
|
75
|
+
eventSource.notify({
|
76
|
+
...finishMetadata,
|
77
|
+
eventType: "finished",
|
78
|
+
result: {
|
79
|
+
status: "success",
|
80
|
+
[outputPropertyName]: result.value,
|
81
|
+
},
|
82
|
+
});
|
83
|
+
return result.value;
|
84
|
+
}
|
85
|
+
exports.executeFunctionCall = executeFunctionCall;
|
@@ -0,0 +1,10 @@
|
|
1
|
+
import { FunctionOptions } from "./FunctionOptions.js";
|
2
|
+
import { FunctionEvent } from "./FunctionEvent.js";
|
3
|
+
export declare function executeFunctionCall<VALUE>({ options, input, functionType, execute, inputPropertyName, outputPropertyName, }: {
|
4
|
+
options?: FunctionOptions;
|
5
|
+
input: unknown;
|
6
|
+
functionType: FunctionEvent["functionType"];
|
7
|
+
execute: (options?: FunctionOptions) => PromiseLike<VALUE>;
|
8
|
+
inputPropertyName?: string;
|
9
|
+
outputPropertyName?: string;
|
10
|
+
}): Promise<VALUE>;
|