modelfusion 0.48.0 → 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/README.md +186 -183
  2. package/core/FunctionEvent.d.ts +9 -1
  3. package/core/FunctionOptions.d.ts +4 -0
  4. package/core/executeFunctionCall.cjs +85 -0
  5. package/core/executeFunctionCall.d.ts +10 -0
  6. package/core/executeFunctionCall.js +81 -0
  7. package/guard/GuardEvent.d.ts +7 -0
  8. package/guard/guard.cjs +60 -54
  9. package/guard/guard.d.ts +2 -1
  10. package/guard/guard.js +60 -54
  11. package/guard/index.cjs +1 -0
  12. package/guard/index.d.ts +1 -0
  13. package/guard/index.js +1 -0
  14. package/model-function/AsyncIterableResultPromise.cjs +5 -5
  15. package/model-function/AsyncIterableResultPromise.d.ts +3 -3
  16. package/model-function/AsyncIterableResultPromise.js +5 -5
  17. package/model-function/Model.d.ts +1 -1
  18. package/model-function/ModelCallEvent.d.ts +5 -7
  19. package/model-function/embed/EmbeddingEvent.d.ts +2 -2
  20. package/model-function/embed/embed.cjs +5 -5
  21. package/model-function/embed/embed.js +5 -5
  22. package/model-function/{executeCall.cjs → executeStandardCall.cjs} +5 -3
  23. package/model-function/{executeCall.d.ts → executeStandardCall.d.ts} +1 -1
  24. package/model-function/{executeCall.js → executeStandardCall.js} +3 -1
  25. package/model-function/executeStreamCall.cjs +134 -0
  26. package/model-function/executeStreamCall.d.ts +20 -0
  27. package/model-function/executeStreamCall.js +130 -0
  28. package/model-function/generate-image/ImageGenerationEvent.d.ts +2 -2
  29. package/model-function/generate-image/generateImage.cjs +3 -3
  30. package/model-function/generate-image/generateImage.js +3 -3
  31. package/model-function/generate-speech/SpeechGenerationEvent.d.ts +27 -0
  32. package/model-function/generate-speech/SpeechGenerationModel.d.ts +15 -0
  33. package/model-function/generate-speech/generateSpeech.cjs +24 -0
  34. package/model-function/generate-speech/generateSpeech.d.ts +8 -0
  35. package/model-function/generate-speech/generateSpeech.js +20 -0
  36. package/model-function/generate-speech/index.cjs +20 -0
  37. package/model-function/generate-speech/index.d.ts +4 -0
  38. package/model-function/generate-speech/index.js +4 -0
  39. package/model-function/generate-speech/streamSpeech.cjs +34 -0
  40. package/model-function/generate-speech/streamSpeech.d.ts +8 -0
  41. package/model-function/generate-speech/streamSpeech.js +30 -0
  42. package/model-function/generate-structure/StructureGenerationEvent.d.ts +2 -2
  43. package/model-function/generate-structure/StructureStreamingEvent.d.ts +2 -2
  44. package/model-function/generate-structure/generateStructure.cjs +3 -3
  45. package/model-function/generate-structure/generateStructure.js +3 -3
  46. package/model-function/generate-structure/generateStructureOrText.cjs +3 -3
  47. package/model-function/generate-structure/generateStructureOrText.js +3 -3
  48. package/model-function/generate-structure/index.cjs +27 -0
  49. package/model-function/generate-structure/index.d.ts +11 -0
  50. package/model-function/generate-structure/index.js +11 -0
  51. package/model-function/generate-structure/streamStructure.cjs +29 -137
  52. package/model-function/generate-structure/streamStructure.js +28 -136
  53. package/model-function/generate-text/TextGenerationEvent.d.ts +8 -2
  54. package/model-function/generate-text/generateText.cjs +4 -4
  55. package/model-function/generate-text/generateText.d.ts +1 -1
  56. package/model-function/generate-text/generateText.js +4 -4
  57. package/model-function/generate-text/index.cjs +0 -1
  58. package/model-function/generate-text/index.d.ts +0 -1
  59. package/model-function/generate-text/index.js +0 -1
  60. package/model-function/generate-text/streamText.cjs +22 -129
  61. package/model-function/generate-text/streamText.js +21 -128
  62. package/model-function/generate-text/trimChatPrompt.cjs +1 -1
  63. package/model-function/generate-text/trimChatPrompt.d.ts +1 -1
  64. package/model-function/generate-text/trimChatPrompt.js +1 -1
  65. package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.d.ts +2 -2
  66. package/model-function/{transcribe-speech/transcribe.cjs → generate-transcription/generateTranscription.cjs} +7 -7
  67. package/model-function/{transcribe-speech/transcribe.d.ts → generate-transcription/generateTranscription.d.ts} +2 -2
  68. package/model-function/{transcribe-speech/transcribe.js → generate-transcription/generateTranscription.js} +5 -5
  69. package/model-function/index.cjs +5 -20
  70. package/model-function/index.d.ts +5 -20
  71. package/model-function/index.js +5 -20
  72. package/model-provider/elevenlabs/{ElevenLabsSpeechSynthesisModel.cjs → ElevenLabsSpeechModel.cjs} +6 -6
  73. package/model-provider/elevenlabs/{ElevenLabsSpeechSynthesisModel.d.ts → ElevenLabsSpeechModel.d.ts} +8 -8
  74. package/model-provider/elevenlabs/{ElevenLabsSpeechSynthesisModel.js → ElevenLabsSpeechModel.js} +4 -4
  75. package/model-provider/elevenlabs/index.cjs +1 -1
  76. package/model-provider/elevenlabs/index.d.ts +1 -1
  77. package/model-provider/elevenlabs/index.js +1 -1
  78. package/model-provider/huggingface/HuggingFaceImageDescriptionModel.cjs +21 -2
  79. package/model-provider/huggingface/HuggingFaceImageDescriptionModel.d.ts +11 -6
  80. package/model-provider/huggingface/HuggingFaceImageDescriptionModel.js +21 -2
  81. package/model-provider/lmnt/{LmntSpeechSynthesisModel.cjs → LmntSpeechModel.cjs} +5 -5
  82. package/model-provider/lmnt/LmntSpeechModel.d.ts +26 -0
  83. package/model-provider/lmnt/{LmntSpeechSynthesisModel.js → LmntSpeechModel.js} +3 -3
  84. package/model-provider/lmnt/index.cjs +1 -1
  85. package/model-provider/lmnt/index.d.ts +1 -1
  86. package/model-provider/lmnt/index.js +1 -1
  87. package/model-provider/openai/{OpenAITextGenerationModel.cjs → OpenAICompletionModel.cjs} +17 -17
  88. package/model-provider/openai/{OpenAITextGenerationModel.d.ts → OpenAICompletionModel.d.ts} +29 -29
  89. package/model-provider/openai/{OpenAITextGenerationModel.js → OpenAICompletionModel.js} +12 -12
  90. package/model-provider/openai/OpenAICostCalculator.cjs +8 -8
  91. package/model-provider/openai/OpenAICostCalculator.js +8 -8
  92. package/model-provider/openai/OpenAITextEmbeddingModel.d.ts +3 -3
  93. package/model-provider/openai/OpenAITranscriptionModel.d.ts +1 -1
  94. package/model-provider/openai/TikTokenTokenizer.d.ts +2 -2
  95. package/model-provider/openai/chat/OpenAIChatModel.d.ts +7 -7
  96. package/model-provider/openai/index.cjs +1 -1
  97. package/model-provider/openai/index.d.ts +1 -1
  98. package/model-provider/openai/index.js +1 -1
  99. package/package.json +1 -1
  100. package/retriever/retrieve.cjs +7 -75
  101. package/retriever/retrieve.js +7 -75
  102. package/tool/UseToolEvent.d.ts +7 -0
  103. package/tool/UseToolOrGenerateTextEvent.d.ts +7 -0
  104. package/tool/executeTool.cjs +2 -0
  105. package/tool/executeTool.js +2 -0
  106. package/tool/index.cjs +2 -0
  107. package/tool/index.d.ts +2 -0
  108. package/tool/index.js +2 -0
  109. package/tool/useTool.cjs +18 -10
  110. package/tool/useTool.js +18 -10
  111. package/tool/useToolOrGenerateText.cjs +34 -26
  112. package/tool/useToolOrGenerateText.js +34 -26
  113. package/vector-index/UpsertIntoVectorIndexEvent.cjs +2 -0
  114. package/vector-index/UpsertIntoVectorIndexEvent.d.ts +9 -0
  115. package/vector-index/UpsertIntoVectorIndexEvent.js +1 -0
  116. package/vector-index/VectorIndexRetriever.cjs +1 -4
  117. package/vector-index/VectorIndexRetriever.js +1 -4
  118. package/vector-index/index.cjs +1 -0
  119. package/vector-index/index.d.ts +1 -0
  120. package/vector-index/index.js +1 -0
  121. package/vector-index/upsertIntoVectorIndex.cjs +16 -7
  122. package/vector-index/upsertIntoVectorIndex.js +16 -7
  123. package/model-function/describe-image/ImageDescriptionEvent.d.ts +0 -18
  124. package/model-function/describe-image/ImageDescriptionModel.d.ts +0 -10
  125. package/model-function/describe-image/describeImage.cjs +0 -26
  126. package/model-function/describe-image/describeImage.d.ts +0 -9
  127. package/model-function/describe-image/describeImage.js +0 -22
  128. package/model-function/generate-text/TextStreamingEvent.d.ts +0 -7
  129. package/model-function/synthesize-speech/SpeechSynthesisEvent.d.ts +0 -21
  130. package/model-function/synthesize-speech/SpeechSynthesisModel.d.ts +0 -15
  131. package/model-function/synthesize-speech/synthesizeSpeech.cjs +0 -67
  132. package/model-function/synthesize-speech/synthesizeSpeech.d.ts +0 -14
  133. package/model-function/synthesize-speech/synthesizeSpeech.js +0 -63
  134. package/model-provider/lmnt/LmntSpeechSynthesisModel.d.ts +0 -26
  135. /package/{model-function/describe-image/ImageDescriptionEvent.cjs → guard/GuardEvent.cjs} +0 -0
  136. /package/{model-function/describe-image/ImageDescriptionEvent.js → guard/GuardEvent.js} +0 -0
  137. /package/model-function/{describe-image/ImageDescriptionModel.cjs → generate-speech/SpeechGenerationEvent.cjs} +0 -0
  138. /package/model-function/{describe-image/ImageDescriptionModel.js → generate-speech/SpeechGenerationEvent.js} +0 -0
  139. /package/model-function/{generate-text/TextStreamingEvent.cjs → generate-speech/SpeechGenerationModel.cjs} +0 -0
  140. /package/model-function/{generate-text/TextStreamingEvent.js → generate-speech/SpeechGenerationModel.js} +0 -0
  141. /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.cjs +0 -0
  142. /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.js +0 -0
  143. /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.cjs +0 -0
  144. /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.d.ts +0 -0
  145. /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.js +0 -0
  146. /package/{model-function/synthesize-speech/SpeechSynthesisEvent.cjs → tool/UseToolEvent.cjs} +0 -0
  147. /package/{model-function/synthesize-speech/SpeechSynthesisEvent.js → tool/UseToolEvent.js} +0 -0
  148. /package/{model-function/synthesize-speech/SpeechSynthesisModel.cjs → tool/UseToolOrGenerateTextEvent.cjs} +0 -0
  149. /package/{model-function/synthesize-speech/SpeechSynthesisModel.js → tool/UseToolOrGenerateTextEvent.js} +0 -0
package/README.md CHANGED
@@ -15,13 +15,14 @@
15
15
 
16
16
  ## Introduction
17
17
 
18
- ModelFusion is a library for building AI apps, chatbots, and agents. It provides abstractions for AI models, vector indices, and tools.
18
+ ModelFusion is a library for building AI applications, chatbots, and agents. Here are the main features:
19
19
 
20
- - **Multimodal Support**: Beyond just LLMs, ModelFusion encompasses a diverse array of models including text generation, text-to-speech, speech-to-text, and image generation, allowing you to build multi-modal AI applications with ease.
21
- - **Flexibility and control**: AI application development can be complex and unique to each project. With ModelFusion, you have complete control over the prompts and model settings, and you can access the raw responses from the models quickly to build what you need.
22
- - **Type inference and validation**: ModelFusion uses TypeScript to infer types wherever possible and to validate model responses. By default, [Zod](https://github.com/colinhacks/zod) is used for type validation, but you can also use other libraries.
23
- - **Guards**: ModelFusion provides a guard function that you can use to implement retry on error, redacting and changing reponses, etc.
24
- - **Integrated support features**: Essential features like **observability**, logging, retries, throttling, tracing, and error handling are built-in, helping you focus more on building your application.
20
+ - **Multimodal**: ModelFusion supports a wide range of models including text generation, image generation, text-to-speech, speech-to-text, and embedding models.
21
+ - **Streaming**: ModelFusion supports streaming for many generation models, e.g. text streaming, structure streaming, and full duplex speech streaming.
22
+ - **Utility functions**: ModelFusion provides functionality for tools and tool usage, vector indices, and guards functions.
23
+ - **Type inference and validation**: ModelFusion infers TypeScript types wherever possible and to validates model responses.
24
+ - **Observability and logging**: ModelFusion provides an observer framework and out-of-the-box logging support.
25
+ - **Resilience and Robustness**: ModelFusion ensures seamless operation through automatic retries, throttling, and error handling mechanisms.
25
26
 
26
27
  ## Quick Install
27
28
 
@@ -35,17 +36,17 @@ Or use a template: [ModelFusion terminal app starter](https://github.com/lgramme
35
36
 
36
37
  You can provide API keys for the different [integrations](https://modelfusion.dev/integration/model-provider/) using environment variables (e.g., `OPENAI_API_KEY`) or pass them into the model constructors as options.
37
38
 
38
- ### [Generate and Stream Text](https://modelfusion.dev/guide/function/generate-text)
39
+ ### [Generate Text](https://modelfusion.dev/guide/function/generate-text)
39
40
 
40
41
  Generate text using a language model and a prompt.
41
42
  You can stream the text if it is supported by the model.
42
- You can use [prompt formats](https://modelfusion.dev/guide/function/generate-text/prompt-format) to change the prompt format of a model.
43
+ You can use [prompt formats](https://modelfusion.dev/guide/function/generate-text#prompt-format) to change the prompt format of a model.
43
44
 
44
45
  #### generateText
45
46
 
46
47
  ```ts
47
48
  const text = await generateText(
48
- new OpenAITextGenerationModel({
49
+ new OpenAICompletionModel({
49
50
  model: "gpt-3.5-turbo-instruct",
50
51
  }),
51
52
  "Write a short story about a robot learning to love:\n\n"
@@ -58,7 +59,7 @@ Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai),
58
59
 
59
60
  ```ts
60
61
  const textStream = await streamText(
61
- new OpenAITextGenerationModel({
62
+ new OpenAICompletionModel({
62
63
  model: "gpt-3.5-turbo-instruct",
63
64
  }),
64
65
  "Write a short story about a robot learning to love:\n\n"
@@ -71,7 +72,85 @@ for await (const textFragment of textStream) {
71
72
 
72
73
  Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp)
73
74
 
74
- ### [Generate and Stream Structure](https://modelfusion.dev/guide/function/generate-structure#generatestructure)
75
+ ### [Generate Image](https://modelfusion.dev/guide/function/generate-image)
76
+
77
+ Generate an image from a prompt.
78
+
79
+ ```ts
80
+ const image = await generateImage(
81
+ new OpenAIImageGenerationModel({ size: "512x512" }),
82
+ "the wicked witch of the west in the style of early 19th century painting"
83
+ );
84
+ ```
85
+
86
+ Providers: [OpenAI (Dall·E)](https://modelfusion.dev/integration/model-provider/openai), [Stability AI](https://modelfusion.dev/integration/model-provider/stability), [Automatic1111](https://modelfusion.dev/integration/model-provider/automatic1111)
87
+
88
+ ### [Generate Speech](https://modelfusion.dev/guide/function/generate-speech)
89
+
90
+ Synthesize speech (audio) from text. Also called TTS (text-to-speech).
91
+
92
+ #### generateSpeech
93
+
94
+ `generateSpeech` synthesizes speech from text.
95
+
96
+ ```ts
97
+ // `speech` is a Buffer with MP3 audio data
98
+ const speech = await generateSpeech(
99
+ new LmntSpeechModel({
100
+ voice: "034b632b-df71-46c8-b440-86a42ffc3cf3", // Henry
101
+ }),
102
+ "Good evening, ladies and gentlemen! Exciting news on the airwaves tonight " +
103
+ "as The Rolling Stones unveil 'Hackney Diamonds,' their first collection of " +
104
+ "fresh tunes in nearly twenty years, featuring the illustrious Lady Gaga, the " +
105
+ "magical Stevie Wonder, and the final beats from the late Charlie Watts."
106
+ );
107
+ ```
108
+
109
+ Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs), [LMNT](https://modelfusion.dev/integration/model-provider/lmnt)
110
+
111
+ #### streamSpeech
112
+
113
+ `generateSpeech` generates a stream of speech chunks from text or from a text stream. Depending on the model, this can be fully duplex.
114
+
115
+ ```ts
116
+ const textStream = await streamText(/* ... */);
117
+
118
+ const speechStream = await streamSpeech(
119
+ new ElevenLabsSpeechModel({
120
+ voice: "pNInz6obpgDQGcFmaJgB", // Adam
121
+ model: "eleven_monolingual_v1",
122
+ voiceSettings: { stability: 1, similarityBoost: 0.35 },
123
+ generationConfig: {
124
+ chunkLengthSchedule: [50, 90, 120, 150, 200],
125
+ },
126
+ }),
127
+ textStream
128
+ );
129
+
130
+ for await (const part of speechStream) {
131
+ // each part is a Buffer with MP3 audio data
132
+ }
133
+ ```
134
+
135
+ Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs)
136
+
137
+ ### [Generate Transcription](https://modelfusion.dev/guide/function/generate-transcription)
138
+
139
+ Transcribe speech (audio) data into text. Also called speech-to-text (STT).
140
+
141
+ ```ts
142
+ const transcription = await generateTranscription(
143
+ new OpenAITranscriptionModel({ model: "whisper-1" }),
144
+ {
145
+ type: "mp3",
146
+ data: await fs.promises.readFile("data/test.mp3"),
147
+ }
148
+ );
149
+ ```
150
+
151
+ Providers: [OpenAI (Whisper)](https://modelfusion.dev/integration/model-provider/openai)
152
+
153
+ ### [Generate Structure](https://modelfusion.dev/guide/function/generate-structure#generatestructure)
75
154
 
76
155
  Generate typed objects using a language model and a schema.
77
156
 
@@ -189,161 +268,6 @@ const { structure, value, text } = await generateStructureOrText(
189
268
 
190
269
  Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai)
191
270
 
192
- ### [Tools](https://modelfusion.dev/guide/tools)
193
-
194
- Tools are functions that can be executed by an AI model. They are useful for building chatbots and agents.
195
-
196
- Predefined tools: [SerpAPI](https://modelfusion.dev/integration/tool/serpapi), [Google Custom Search](https://modelfusion.dev/integration/tool/google-custom-search)
197
-
198
- #### Create Tool
199
-
200
- A tool is a function with a name, a description, and a schema for the input parameters.
201
-
202
- ```ts
203
- const calculator = new Tool({
204
- name: "calculator",
205
- description: "Execute a calculation",
206
-
207
- inputSchema: new ZodSchema(
208
- z.object({
209
- a: z.number().describe("The first number."),
210
- b: z.number().describe("The second number."),
211
- operator: z
212
- .enum(["+", "-", "*", "/"])
213
- .describe("The operator (+, -, *, /)."),
214
- })
215
- ),
216
-
217
- execute: async ({ a, b, operator }) => {
218
- switch (operator) {
219
- case "+":
220
- return a + b;
221
- case "-":
222
- return a - b;
223
- case "*":
224
- return a * b;
225
- case "/":
226
- return a / b;
227
- default:
228
- throw new Error(`Unknown operator: ${operator}`);
229
- }
230
- },
231
- });
232
- ```
233
-
234
- #### useTool
235
-
236
- The model determines the parameters for the tool from the prompt and then executes it.
237
-
238
- ```ts
239
- const { tool, parameters, result } = await useTool(
240
- new OpenAIChatModel({ model: "gpt-3.5-turbo" }),
241
- calculator,
242
- [OpenAIChatMessage.user("What's fourteen times twelve?")]
243
- );
244
- ```
245
-
246
- #### useToolOrGenerateText
247
-
248
- The model determines which tool to use and its parameters from the prompt and then executes it.
249
- Text is generated as a fallback.
250
-
251
- ```ts
252
- const { tool, parameters, result, text } = await useToolOrGenerateText(
253
- new OpenAIChatModel({ model: "gpt-3.5-turbo" }),
254
- [calculator /* and other tools... */],
255
- [OpenAIChatMessage.user("What's fourteen times twelve?")]
256
- );
257
- ```
258
-
259
- ### [Transcribe Speech](https://modelfusion.dev/guide/function/transcribe-speech)
260
-
261
- Turn speech (audio) into text.
262
-
263
- ```ts
264
- const transcription = await transcribe(
265
- new OpenAITranscriptionModel({ model: "whisper-1" }),
266
- {
267
- type: "mp3",
268
- data: await fs.promises.readFile("data/test.mp3"),
269
- }
270
- );
271
- ```
272
-
273
- Providers: [OpenAI (Whisper)](https://modelfusion.dev/integration/model-provider/openai)
274
-
275
- ### [Synthesize Speech](https://modelfusion.dev/guide/function/synthesize-speech)
276
-
277
- Generate speech (audio) from text. Also called TTS (text-to-speech).
278
-
279
- Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs), [LMNT](https://modelfusion.dev/integration/model-provider/lmnt)
280
-
281
- #### Standard mode
282
-
283
- ```ts
284
- // `speech` is a Buffer with MP3 audio data
285
- const speech = await synthesizeSpeech(
286
- new LmntSpeechSynthesisModel({
287
- voice: "034b632b-df71-46c8-b440-86a42ffc3cf3", // Henry
288
- }),
289
- "Good evening, ladies and gentlemen! Exciting news on the airwaves tonight " +
290
- "as The Rolling Stones unveil 'Hackney Diamonds,' their first collection of " +
291
- "fresh tunes in nearly twenty years, featuring the illustrious Lady Gaga, the " +
292
- "magical Stevie Wonder, and the final beats from the late Charlie Watts."
293
- );
294
- ```
295
-
296
- #### Duplex streaming mode
297
-
298
- ```ts
299
- const textStream = await streamText(/* ... */);
300
-
301
- const speechStream = await synthesizeSpeech(
302
- new ElevenLabsSpeechSynthesisModel({
303
- voice: "pNInz6obpgDQGcFmaJgB", // Adam
304
- model: "eleven_monolingual_v1",
305
- voiceSettings: { stability: 1, similarityBoost: 0.35 },
306
- generationConfig: {
307
- chunkLengthSchedule: [50, 90, 120, 150, 200],
308
- },
309
- }),
310
- textStream,
311
- { mode: "stream-duplex" }
312
- );
313
-
314
- for await (const part of speechStream) {
315
- // each part is a Buffer with MP3 audio data
316
- }
317
- ```
318
-
319
- ### [Describe Image](https://modelfusion.dev/guide/function/describe-image)
320
-
321
- Describe an image as text, e.g. for image captioning or OCR.
322
-
323
- ```ts
324
- const text = await describeImage(
325
- new HuggingFaceImageDescriptionModel({
326
- model: "nlpconnect/vit-gpt2-image-captioning",
327
- }),
328
- data // buffer with image data
329
- );
330
- ```
331
-
332
- Providers: [HuggingFace](/integration/model-provider/huggingface)
333
-
334
- ### [Generate Image](https://modelfusion.dev/guide/function/generate-image)
335
-
336
- Generate an image from a prompt.
337
-
338
- ```ts
339
- const image = await generateImage(
340
- new OpenAIImageGenerationModel({ size: "512x512" }),
341
- "the wicked witch of the west in the style of early 19th century painting"
342
- );
343
- ```
344
-
345
- Providers: [OpenAI (Dall·E)](https://modelfusion.dev/integration/model-provider/openai), [Stability AI](https://modelfusion.dev/integration/model-provider/stability), [Automatic1111](https://modelfusion.dev/integration/model-provider/automatic1111)
346
-
347
271
  ### [Embed Value](https://modelfusion.dev/guide/function/embed)
348
272
 
349
273
  Create embeddings for text and other values. Embeddings are vectors that represent the essence of the values in the context of the model.
@@ -393,7 +317,7 @@ Guard functions can be used to implement retry on error, redacting and changing
393
317
 
394
318
  ```ts
395
319
  const result = await guard(
396
- (input) =>
320
+ (input, options) =>
397
321
  generateStructure(
398
322
  new OpenAIChatModel({
399
323
  // ...
@@ -401,7 +325,8 @@ const result = await guard(
401
325
  new ZodStructureDefinition({
402
326
  // ...
403
327
  }),
404
- input
328
+ input,
329
+ options
405
330
  ),
406
331
  [
407
332
  // ...
@@ -420,7 +345,74 @@ const result = await guard(
420
345
  );
421
346
  ```
422
347
 
423
- ### [Upserting and Retrieving Objects from Vector Indices](https://modelfusion.dev/guide/vector-index)
348
+ ### [Tools](https://modelfusion.dev/guide/tools)
349
+
350
+ Tools are functions that can be executed by an AI model. They are useful for building chatbots and agents.
351
+
352
+ Predefined tools: [SerpAPI](https://modelfusion.dev/integration/tool/serpapi), [Google Custom Search](https://modelfusion.dev/integration/tool/google-custom-search)
353
+
354
+ #### Create Tool
355
+
356
+ A tool is a function with a name, a description, and a schema for the input parameters.
357
+
358
+ ```ts
359
+ const calculator = new Tool({
360
+ name: "calculator",
361
+ description: "Execute a calculation",
362
+
363
+ inputSchema: new ZodSchema(
364
+ z.object({
365
+ a: z.number().describe("The first number."),
366
+ b: z.number().describe("The second number."),
367
+ operator: z
368
+ .enum(["+", "-", "*", "/"])
369
+ .describe("The operator (+, -, *, /)."),
370
+ })
371
+ ),
372
+
373
+ execute: async ({ a, b, operator }) => {
374
+ switch (operator) {
375
+ case "+":
376
+ return a + b;
377
+ case "-":
378
+ return a - b;
379
+ case "*":
380
+ return a * b;
381
+ case "/":
382
+ return a / b;
383
+ default:
384
+ throw new Error(`Unknown operator: ${operator}`);
385
+ }
386
+ },
387
+ });
388
+ ```
389
+
390
+ #### useTool
391
+
392
+ The model determines the parameters for the tool from the prompt and then executes it.
393
+
394
+ ```ts
395
+ const { tool, parameters, result } = await useTool(
396
+ new OpenAIChatModel({ model: "gpt-3.5-turbo" }),
397
+ calculator,
398
+ [OpenAIChatMessage.user("What's fourteen times twelve?")]
399
+ );
400
+ ```
401
+
402
+ #### useToolOrGenerateText
403
+
404
+ The model determines which tool to use and its parameters from the prompt and then executes it.
405
+ Text is generated as a fallback.
406
+
407
+ ```ts
408
+ const { tool, parameters, result, text } = await useToolOrGenerateText(
409
+ new OpenAIChatModel({ model: "gpt-3.5-turbo" }),
410
+ [calculator /* and other tools... */],
411
+ [OpenAIChatMessage.user("What's fourteen times twelve?")]
412
+ );
413
+ ```
414
+
415
+ ### [Vector Indices](https://modelfusion.dev/guide/vector-index)
424
416
 
425
417
  ```ts
426
418
  const texts = [
@@ -460,7 +452,7 @@ Available Vector Stores: [Memory](https://modelfusion.dev/integration/vector-ind
460
452
 
461
453
  Prompt formats let you use higher level prompt structures (such as instruction or chat prompts) for different models.
462
454
 
463
- #### [Text Generation Prompt Formats](https://modelfusion.dev/guide/function/generate-text/prompt-format)
455
+ #### [Text Generation Prompt Formats](https://modelfusion.dev/guide/function/generate-text#prompt-format)
464
456
 
465
457
  ```ts
466
458
  const text = await generateText(
@@ -525,7 +517,7 @@ ModelFusion model functions return rich results that include the original respon
525
517
  ```ts
526
518
  // access the full response (needs to be typed) and the metadata:
527
519
  const { value, response, metadata } = await generateText(
528
- new OpenAITextGenerationModel({
520
+ new OpenAICompletionModel({
529
521
  model: "gpt-3.5-turbo-instruct",
530
522
  maxCompletionTokens: 1000,
531
523
  n: 2, // generate 2 completions
@@ -536,31 +528,34 @@ const { value, response, metadata } = await generateText(
536
528
  console.log(metadata);
537
529
 
538
530
  // cast to the response type:
539
- for (const choice of (response as OpenAITextGenerationResponse).choices) {
531
+ for (const choice of (response as OpenAICompletionResponse).choices) {
540
532
  console.log(choice.text);
541
533
  }
542
534
  ```
543
535
 
544
- ### Observability
536
+ ### Logging and Observability
545
537
 
546
- Integrations: [Helicone](https://modelfusion.dev/integration/observability/helicone)
538
+ ModelFusion provides an [observer framework](https://modelfusion.dev/guide/util/observer) and [out-of-the-box logging support](https://modelfusion.dev/guide/util/logging). You can easily trace runs and call hierarchies, and you can add your own observers.
539
+
540
+ #### Global Logging Example
541
+
542
+ ```ts
543
+ setGlobalFunctionLogging("detailed-object"); // log full events
544
+ ```
547
545
 
548
546
  ## Documentation
549
547
 
550
548
  ### [Guide](https://modelfusion.dev/guide)
551
549
 
552
550
  - [Model Functions](https://modelfusion.dev/guide/function/)
553
- - [Generate and stream text](https://modelfusion.dev/guide/function/generate-text)
554
- - [Prompt Format](https://modelfusion.dev/guide/function/generate-text/prompt-format)
555
- - [Generate and stream structure](https://modelfusion.dev/guide/function/generate-structure)
551
+ - [Generate text](https://modelfusion.dev/guide/function/generate-text)
552
+ - [Generate image](https://modelfusion.dev/guide/function/generate-image)
553
+ - [Generate speech](https://modelfusion.dev/guide/function/generate-speech)
554
+ - [Generate transcription](https://modelfusion.dev/guide/function/generation-transcription)
555
+ - [Generate structure](https://modelfusion.dev/guide/function/generate-structure)
556
556
  - [Generate structure or text](https://modelfusion.dev/guide/function/generate-structure-or-text)
557
557
  - [Tokenize Text](https://modelfusion.dev/guide/function/tokenize-text)
558
558
  - [Embed Value](https://modelfusion.dev/guide/function/embed)
559
- - [Transcribe Speech](https://modelfusion.dev/guide/function/transcribe-speech)
560
- - [Synthesize Speech](https://modelfusion.dev/guide/function/synthesize-speech)
561
- - [Describe Image](https://modelfusion.dev/guide/function/describe-image)
562
- - [Generate Image](https://modelfusion.dev/guide/function/generate-image)
563
- - [Prompt Format](https://modelfusion.dev/guide/function/generate-image/prompt-format)
564
559
  - [Guards](https://modelfusion.dev/guide/guard)
565
560
  - [Tools](https://modelfusion.dev/guide/tools)
566
561
  - [Vector Indices](https://modelfusion.dev/guide/vector-index)
@@ -628,7 +623,7 @@ Create an 19th century painting image for your input.
628
623
 
629
624
  Record audio with push-to-talk and transcribe it using Whisper, implemented as a Next.js app. The app shows a list of the transcriptions.
630
625
 
631
- ### [Duplex Speech Streaming (Vite(React) + Fastify))](https://github.com/lgrammel/modelfusion/tree/main/examples/duplex-speech-streaming-vite-react-fastify)
626
+ ### [Speech Streaming (Vite(React) + Fastify))](https://github.com/lgrammel/modelfusion/tree/main/examples/speech-streaming-vite-react-fastify)
632
627
 
633
628
  > _Speech Streaming_, _OpenAI_, _Elevenlabs_ _streaming_, _Vite_, _Fastify_
634
629
 
@@ -669,3 +664,11 @@ Generate text on a Cloudflare Worker using ModelFusion and OpenAI.
669
664
  ### [Contributing Guide](https://github.com/lgrammel/modelfusion/blob/main/CONTRIBUTING.md)
670
665
 
671
666
  Read the [ModelFusion contributing guide](https://github.com/lgrammel/modelfusion/blob/main/CONTRIBUTING.md) to learn about the development process, how to propose bugfixes and improvements, and how to build and test your changes.
667
+
668
+ ```
669
+
670
+ ```
671
+
672
+ ```
673
+
674
+ ```
@@ -1,11 +1,19 @@
1
+ import { GuardFinishedEvent, GuardStartedEvent } from "../guard/GuardEvent.js";
1
2
  import { ModelCallFinishedEvent, ModelCallStartedEvent } from "../model-function/ModelCallEvent.js";
2
3
  import { RetrieveFinishedEvent, RetrieveStartedEvent } from "../retriever/RetrieveEvent.js";
3
4
  import { ExecuteToolFinishedEvent, ExecuteToolStartedEvent } from "../tool/ExecuteToolEvent.js";
5
+ import { UseToolFinishedEvent, UseToolStartedEvent } from "../tool/UseToolEvent.js";
6
+ import { UseToolOrGenerateTextFinishedEvent, UseToolOrGenerateTextStartedEvent } from "../tool/UseToolOrGenerateTextEvent.js";
7
+ import { UpsertIntoVectorIndexFinishedEvent, UpsertIntoVectorIndexStartedEvent } from "../vector-index/UpsertIntoVectorIndexEvent.js";
4
8
  export interface BaseFunctionEvent {
5
9
  /**
6
10
  * Unique identifier for the function call.
7
11
  */
8
12
  callId: string | undefined;
13
+ /**
14
+ * Unique identifier of the call id of the parent function.
15
+ */
16
+ parentCallId?: string | undefined;
9
17
  /**
10
18
  * Optional unique identifier for the function.
11
19
  */
@@ -73,4 +81,4 @@ export interface BaseFunctionFinishedEvent extends BaseFunctionEvent {
73
81
  */
74
82
  result: BaseFunctionFinishedEventResult;
75
83
  }
76
- export type FunctionEvent = ModelCallStartedEvent | ExecuteToolStartedEvent | RetrieveStartedEvent | ModelCallFinishedEvent | ExecuteToolFinishedEvent | RetrieveFinishedEvent;
84
+ export type FunctionEvent = ExecuteToolStartedEvent | ExecuteToolFinishedEvent | GuardStartedEvent | GuardFinishedEvent | ModelCallStartedEvent | ModelCallFinishedEvent | RetrieveStartedEvent | RetrieveFinishedEvent | UpsertIntoVectorIndexStartedEvent | UpsertIntoVectorIndexFinishedEvent | UseToolStartedEvent | UseToolFinishedEvent | UseToolOrGenerateTextStartedEvent | UseToolOrGenerateTextFinishedEvent;
@@ -21,6 +21,10 @@ export type FunctionOptions = {
21
21
  * Optional run as part of which this function is called.
22
22
  */
23
23
  run?: Run;
24
+ /**
25
+ * Unique identifier of the call id of the parent function.
26
+ */
27
+ parentCallId?: string | undefined;
24
28
  };
25
29
  /**
26
30
  * The logging to use for the function. Logs are sent to the console.
@@ -0,0 +1,85 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.executeFunctionCall = void 0;
4
+ const nanoid_1 = require("nanoid");
5
+ const FunctionEventSource_js_1 = require("./FunctionEventSource.cjs");
6
+ const GlobalFunctionLogging_js_1 = require("./GlobalFunctionLogging.cjs");
7
+ const GlobalFunctionObservers_js_1 = require("./GlobalFunctionObservers.cjs");
8
+ const AbortError_js_1 = require("./api/AbortError.cjs");
9
+ const getFunctionCallLogger_js_1 = require("./getFunctionCallLogger.cjs");
10
+ const getRun_js_1 = require("./getRun.cjs");
11
+ const DurationMeasurement_js_1 = require("../util/DurationMeasurement.cjs");
12
+ const runSafe_js_1 = require("../util/runSafe.cjs");
13
+ async function executeFunctionCall({ options, input, functionType, execute, inputPropertyName = "input", outputPropertyName = "value", }) {
14
+ const run = await (0, getRun_js_1.getRun)(options?.run);
15
+ const eventSource = new FunctionEventSource_js_1.FunctionEventSource({
16
+ observers: [
17
+ ...(0, getFunctionCallLogger_js_1.getFunctionCallLogger)(options?.logging ?? (0, GlobalFunctionLogging_js_1.getGlobalFunctionLogging)()),
18
+ ...(0, GlobalFunctionObservers_js_1.getGlobalFunctionObservers)(),
19
+ ...(run?.functionObserver != null ? [run.functionObserver] : []),
20
+ ...(options?.observers ?? []),
21
+ ],
22
+ errorHandler: run?.errorHandler,
23
+ });
24
+ const durationMeasurement = (0, DurationMeasurement_js_1.startDurationMeasurement)();
25
+ const startMetadata = {
26
+ functionType,
27
+ callId: `call-${(0, nanoid_1.nanoid)()}`,
28
+ parentCallId: options?.parentCallId,
29
+ runId: run?.runId,
30
+ sessionId: run?.sessionId,
31
+ userId: run?.userId,
32
+ functionId: options?.functionId,
33
+ [inputPropertyName]: input,
34
+ timestamp: durationMeasurement.startDate,
35
+ startTimestamp: durationMeasurement.startDate,
36
+ };
37
+ eventSource.notify({
38
+ eventType: "started",
39
+ ...startMetadata,
40
+ });
41
+ const result = await (0, runSafe_js_1.runSafe)(() => execute({
42
+ functionId: options?.functionId,
43
+ logging: options?.logging,
44
+ observers: options?.observers,
45
+ run,
46
+ parentCallId: startMetadata.callId,
47
+ }));
48
+ const finishMetadata = {
49
+ eventType: "finished",
50
+ ...startMetadata,
51
+ finishTimestamp: new Date(),
52
+ durationInMs: durationMeasurement.durationInMs,
53
+ };
54
+ if (!result.ok) {
55
+ if (result.isAborted) {
56
+ eventSource.notify({
57
+ ...finishMetadata,
58
+ eventType: "finished",
59
+ result: {
60
+ status: "abort",
61
+ },
62
+ });
63
+ throw new AbortError_js_1.AbortError();
64
+ }
65
+ eventSource.notify({
66
+ ...finishMetadata,
67
+ eventType: "finished",
68
+ result: {
69
+ status: "error",
70
+ error: result.error,
71
+ },
72
+ });
73
+ throw result.error;
74
+ }
75
+ eventSource.notify({
76
+ ...finishMetadata,
77
+ eventType: "finished",
78
+ result: {
79
+ status: "success",
80
+ [outputPropertyName]: result.value,
81
+ },
82
+ });
83
+ return result.value;
84
+ }
85
+ exports.executeFunctionCall = executeFunctionCall;
@@ -0,0 +1,10 @@
1
+ import { FunctionOptions } from "./FunctionOptions.js";
2
+ import { FunctionEvent } from "./FunctionEvent.js";
3
+ export declare function executeFunctionCall<VALUE>({ options, input, functionType, execute, inputPropertyName, outputPropertyName, }: {
4
+ options?: FunctionOptions;
5
+ input: unknown;
6
+ functionType: FunctionEvent["functionType"];
7
+ execute: (options?: FunctionOptions) => PromiseLike<VALUE>;
8
+ inputPropertyName?: string;
9
+ outputPropertyName?: string;
10
+ }): Promise<VALUE>;