modelfusion 0.47.3 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -33
- package/core/getRun.cjs +5 -3
- package/core/getRun.js +5 -3
- package/index.cjs +1 -0
- package/index.d.ts +1 -0
- package/index.js +1 -0
- package/model-function/AsyncIterableResultPromise.cjs +5 -5
- package/model-function/AsyncIterableResultPromise.d.ts +3 -3
- package/model-function/AsyncIterableResultPromise.js +5 -5
- package/model-function/Model.d.ts +1 -1
- package/model-function/ModelCallEvent.d.ts +5 -7
- package/model-function/embed/embed.cjs +3 -3
- package/model-function/embed/embed.js +3 -3
- package/model-function/{executeCall.cjs → executeStandardCall.cjs} +3 -3
- package/model-function/{executeCall.d.ts → executeStandardCall.d.ts} +1 -1
- package/model-function/{executeCall.js → executeStandardCall.js} +1 -1
- package/model-function/executeStreamCall.cjs +132 -0
- package/model-function/executeStreamCall.d.ts +20 -0
- package/model-function/executeStreamCall.js +128 -0
- package/model-function/generate-image/generateImage.cjs +2 -2
- package/model-function/generate-image/generateImage.js +2 -2
- package/model-function/generate-speech/SpeechGenerationEvent.d.ts +27 -0
- package/model-function/generate-speech/SpeechGenerationModel.d.ts +15 -0
- package/model-function/{synthesize-speech/synthesizeSpeech.cjs → generate-speech/generateSpeech.cjs} +7 -7
- package/model-function/{synthesize-speech/synthesizeSpeech.d.ts → generate-speech/generateSpeech.d.ts} +2 -2
- package/model-function/{synthesize-speech/synthesizeSpeech.js → generate-speech/generateSpeech.js} +5 -5
- package/model-function/generate-speech/index.cjs +20 -0
- package/model-function/generate-speech/index.d.ts +4 -0
- package/model-function/generate-speech/index.js +4 -0
- package/model-function/generate-speech/streamSpeech.cjs +34 -0
- package/model-function/generate-speech/streamSpeech.d.ts +8 -0
- package/model-function/generate-speech/streamSpeech.js +30 -0
- package/model-function/generate-structure/generateStructure.cjs +2 -2
- package/model-function/generate-structure/generateStructure.js +2 -2
- package/model-function/generate-structure/generateStructureOrText.cjs +2 -2
- package/model-function/generate-structure/generateStructureOrText.js +2 -2
- package/model-function/generate-structure/index.cjs +27 -0
- package/model-function/generate-structure/index.d.ts +11 -0
- package/model-function/generate-structure/index.js +11 -0
- package/model-function/generate-structure/streamStructure.cjs +28 -136
- package/model-function/generate-structure/streamStructure.js +27 -135
- package/model-function/generate-text/TextGenerationEvent.d.ts +6 -0
- package/model-function/generate-text/generateText.cjs +3 -3
- package/model-function/generate-text/generateText.d.ts +1 -1
- package/model-function/generate-text/generateText.js +3 -3
- package/model-function/generate-text/index.cjs +0 -1
- package/model-function/generate-text/index.d.ts +0 -1
- package/model-function/generate-text/index.js +0 -1
- package/model-function/generate-text/streamText.cjs +21 -128
- package/model-function/generate-text/streamText.js +20 -127
- package/model-function/generate-text/trimChatPrompt.cjs +1 -1
- package/model-function/generate-text/trimChatPrompt.d.ts +1 -1
- package/model-function/generate-text/trimChatPrompt.js +1 -1
- package/model-function/{transcribe-speech/transcribe.cjs → generate-transcription/generateTranscription.cjs} +6 -6
- package/model-function/{transcribe-speech/transcribe.d.ts → generate-transcription/generateTranscription.d.ts} +2 -2
- package/model-function/{transcribe-speech/transcribe.js → generate-transcription/generateTranscription.js} +4 -4
- package/model-function/index.cjs +5 -20
- package/model-function/index.d.ts +5 -20
- package/model-function/index.js +5 -20
- package/model-provider/elevenlabs/ElevenLabsApiConfiguration.cjs +3 -0
- package/model-provider/elevenlabs/ElevenLabsApiConfiguration.d.ts +1 -0
- package/model-provider/elevenlabs/ElevenLabsApiConfiguration.js +3 -0
- package/model-provider/elevenlabs/ElevenLabsSpeechModel.cjs +191 -0
- package/model-provider/elevenlabs/ElevenLabsSpeechModel.d.ts +39 -0
- package/model-provider/elevenlabs/ElevenLabsSpeechModel.js +187 -0
- package/model-provider/elevenlabs/index.cjs +1 -1
- package/model-provider/elevenlabs/index.d.ts +1 -1
- package/model-provider/elevenlabs/index.js +1 -1
- package/model-provider/huggingface/HuggingFaceImageDescriptionModel.cjs +21 -2
- package/model-provider/huggingface/HuggingFaceImageDescriptionModel.d.ts +11 -6
- package/model-provider/huggingface/HuggingFaceImageDescriptionModel.js +21 -2
- package/model-provider/lmnt/{LmntSpeechSynthesisModel.cjs → LmntSpeechModel.cjs} +5 -5
- package/model-provider/lmnt/LmntSpeechModel.d.ts +26 -0
- package/model-provider/lmnt/{LmntSpeechSynthesisModel.js → LmntSpeechModel.js} +3 -3
- package/model-provider/lmnt/index.cjs +1 -1
- package/model-provider/lmnt/index.d.ts +1 -1
- package/model-provider/lmnt/index.js +1 -1
- package/model-provider/openai/{OpenAITextGenerationModel.cjs → OpenAICompletionModel.cjs} +17 -17
- package/model-provider/openai/{OpenAITextGenerationModel.d.ts → OpenAICompletionModel.d.ts} +25 -25
- package/model-provider/openai/{OpenAITextGenerationModel.js → OpenAICompletionModel.js} +12 -12
- package/model-provider/openai/OpenAICostCalculator.cjs +3 -3
- package/model-provider/openai/OpenAICostCalculator.js +3 -3
- package/model-provider/openai/OpenAITranscriptionModel.d.ts +1 -1
- package/model-provider/openai/TikTokenTokenizer.d.ts +2 -2
- package/model-provider/openai/index.cjs +1 -1
- package/model-provider/openai/index.d.ts +1 -1
- package/model-provider/openai/index.js +1 -1
- package/package.json +3 -1
- package/ui/MediaSourceAppender.cjs +54 -0
- package/ui/MediaSourceAppender.d.ts +11 -0
- package/ui/MediaSourceAppender.js +50 -0
- package/ui/index.cjs +17 -0
- package/ui/index.d.ts +1 -0
- package/ui/index.js +1 -0
- package/util/SimpleWebSocket.cjs +41 -0
- package/util/SimpleWebSocket.d.ts +12 -0
- package/util/SimpleWebSocket.js +14 -0
- package/model-function/describe-image/ImageDescriptionEvent.d.ts +0 -18
- package/model-function/describe-image/ImageDescriptionModel.d.ts +0 -10
- package/model-function/describe-image/describeImage.cjs +0 -26
- package/model-function/describe-image/describeImage.d.ts +0 -9
- package/model-function/describe-image/describeImage.js +0 -22
- package/model-function/generate-text/TextStreamingEvent.cjs +0 -2
- package/model-function/generate-text/TextStreamingEvent.d.ts +0 -7
- package/model-function/generate-text/TextStreamingEvent.js +0 -1
- package/model-function/synthesize-speech/SpeechSynthesisEvent.cjs +0 -2
- package/model-function/synthesize-speech/SpeechSynthesisEvent.d.ts +0 -21
- package/model-function/synthesize-speech/SpeechSynthesisEvent.js +0 -1
- package/model-function/synthesize-speech/SpeechSynthesisModel.cjs +0 -2
- package/model-function/synthesize-speech/SpeechSynthesisModel.d.ts +0 -11
- package/model-function/synthesize-speech/SpeechSynthesisModel.js +0 -1
- package/model-provider/elevenlabs/ElevenLabsSpeechSynthesisModel.cjs +0 -79
- package/model-provider/elevenlabs/ElevenLabsSpeechSynthesisModel.d.ts +0 -30
- package/model-provider/elevenlabs/ElevenLabsSpeechSynthesisModel.js +0 -75
- package/model-provider/lmnt/LmntSpeechSynthesisModel.d.ts +0 -26
- /package/model-function/{describe-image/ImageDescriptionEvent.cjs → generate-speech/SpeechGenerationEvent.cjs} +0 -0
- /package/model-function/{describe-image/ImageDescriptionEvent.js → generate-speech/SpeechGenerationEvent.js} +0 -0
- /package/model-function/{describe-image/ImageDescriptionModel.cjs → generate-speech/SpeechGenerationModel.cjs} +0 -0
- /package/model-function/{describe-image/ImageDescriptionModel.js → generate-speech/SpeechGenerationModel.js} +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.cjs +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.d.ts +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.js +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.cjs +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.d.ts +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.js +0 -0
@@ -0,0 +1,187 @@
|
|
1
|
+
import { z } from "zod";
|
2
|
+
import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
|
3
|
+
import { createAudioMpegResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
|
4
|
+
import { AsyncQueue } from "../../event-source/AsyncQueue.js";
|
5
|
+
import { AbstractModel } from "../../model-function/AbstractModel.js";
|
6
|
+
import { createSimpleWebSocket } from "../../util/SimpleWebSocket.js";
|
7
|
+
import { safeParseJsonWithZod } from "../../util/parseJSON.js";
|
8
|
+
import { ElevenLabsApiConfiguration } from "./ElevenLabsApiConfiguration.js";
|
9
|
+
import { failedElevenLabsCallResponseHandler } from "./ElevenLabsError.js";
|
10
|
+
const elevenLabsModels = [
|
11
|
+
"eleven_multilingual_v2",
|
12
|
+
"eleven_multilingual_v1",
|
13
|
+
"eleven_monolingual_v1",
|
14
|
+
];
|
15
|
+
const defaultModel = "eleven_multilingual_v2";
|
16
|
+
/**
|
17
|
+
* Synthesize speech using the ElevenLabs Text to Speech API.
|
18
|
+
*
|
19
|
+
* @see https://api.elevenlabs.io/docs#/text-to-speech/Text_to_speech_v1_text_to_speech__voice_id__post
|
20
|
+
*/
|
21
|
+
export class ElevenLabsSpeechModel extends AbstractModel {
|
22
|
+
constructor(settings) {
|
23
|
+
super({ settings });
|
24
|
+
Object.defineProperty(this, "provider", {
|
25
|
+
enumerable: true,
|
26
|
+
configurable: true,
|
27
|
+
writable: true,
|
28
|
+
value: "elevenlabs"
|
29
|
+
});
|
30
|
+
}
|
31
|
+
get modelName() {
|
32
|
+
return this.settings.voice;
|
33
|
+
}
|
34
|
+
async callAPI(text, options) {
|
35
|
+
return callWithRetryAndThrottle({
|
36
|
+
retry: this.settings.api?.retry,
|
37
|
+
throttle: this.settings.api?.throttle,
|
38
|
+
call: async () => callElevenLabsTextToSpeechAPI({
|
39
|
+
api: this.settings.api,
|
40
|
+
abortSignal: options?.run?.abortSignal,
|
41
|
+
text,
|
42
|
+
voiceId: this.settings.voice,
|
43
|
+
modelId: this.settings.model,
|
44
|
+
voiceSettings: this.settings.voiceSettings,
|
45
|
+
}),
|
46
|
+
});
|
47
|
+
}
|
48
|
+
get settingsForEvent() {
|
49
|
+
return {
|
50
|
+
model: this.settings.model,
|
51
|
+
voice: this.settings.voice,
|
52
|
+
voiceSettings: this.settings.voiceSettings,
|
53
|
+
};
|
54
|
+
}
|
55
|
+
doGenerateSpeechStandard(text, options) {
|
56
|
+
return this.callAPI(text, options);
|
57
|
+
}
|
58
|
+
async doGenerateSpeechStreamDuplex(textStream
|
59
|
+
// options?: FunctionOptions | undefined
|
60
|
+
) {
|
61
|
+
const responseSchema = z.union([
|
62
|
+
z.object({
|
63
|
+
audio: z.string(),
|
64
|
+
isFinal: z.literal(false).nullable(),
|
65
|
+
normalizedAlignment: z
|
66
|
+
.object({
|
67
|
+
chars: z.array(z.string()),
|
68
|
+
charStartTimesMs: z.array(z.number()),
|
69
|
+
charDurationsMs: z.array(z.number()),
|
70
|
+
})
|
71
|
+
.nullable(),
|
72
|
+
}),
|
73
|
+
z.object({
|
74
|
+
isFinal: z.literal(true),
|
75
|
+
}),
|
76
|
+
z.object({
|
77
|
+
message: z.string(),
|
78
|
+
error: z.string(),
|
79
|
+
code: z.number(),
|
80
|
+
}),
|
81
|
+
]);
|
82
|
+
const queue = new AsyncQueue();
|
83
|
+
const model = this.settings.model ?? defaultModel;
|
84
|
+
const socket = await createSimpleWebSocket(`wss://api.elevenlabs.io/v1/text-to-speech/${this.settings.voice}/stream-input?model_id=${model}`);
|
85
|
+
socket.onopen = async () => {
|
86
|
+
const api = this.settings.api ?? new ElevenLabsApiConfiguration();
|
87
|
+
// send begin-of-stream (BOS) message:
|
88
|
+
socket.send(JSON.stringify({
|
89
|
+
// The JS WebSocket API does not support authorization headers, so we send the API key in the BOS message.
|
90
|
+
// See https://stackoverflow.com/questions/4361173/http-headers-in-websockets-client-api
|
91
|
+
xi_api_key: api.apiKey,
|
92
|
+
text: " ",
|
93
|
+
voice_settings: toApiVoiceSettings(this.settings.voiceSettings),
|
94
|
+
generation_config: toGenerationConfig(this.settings.generationConfig),
|
95
|
+
}));
|
96
|
+
// send text in chunks:
|
97
|
+
let textBuffer = "";
|
98
|
+
for await (const textDelta of textStream) {
|
99
|
+
textBuffer += textDelta;
|
100
|
+
// using ". " as separator: sending in full sentences improves the quality
|
101
|
+
// of the audio output significantly.
|
102
|
+
const separator = textBuffer.lastIndexOf(". ");
|
103
|
+
if (separator === -1) {
|
104
|
+
continue;
|
105
|
+
}
|
106
|
+
const textToProcess = textBuffer.slice(0, separator);
|
107
|
+
textBuffer = textBuffer.slice(separator + 1);
|
108
|
+
socket.send(JSON.stringify({
|
109
|
+
text: textToProcess,
|
110
|
+
try_trigger_generation: true,
|
111
|
+
}));
|
112
|
+
}
|
113
|
+
// send remaining text:
|
114
|
+
if (textBuffer.length > 0) {
|
115
|
+
socket.send(JSON.stringify({
|
116
|
+
text: `${textBuffer} `,
|
117
|
+
try_trigger_generation: true,
|
118
|
+
}));
|
119
|
+
}
|
120
|
+
// send end-of-stream (EOS) message:
|
121
|
+
socket.send(JSON.stringify({ text: "" }));
|
122
|
+
};
|
123
|
+
socket.onmessage = (event) => {
|
124
|
+
const parseResult = safeParseJsonWithZod(event.data, responseSchema);
|
125
|
+
if (!parseResult.success) {
|
126
|
+
queue.push({ type: "error", error: parseResult.error });
|
127
|
+
return;
|
128
|
+
}
|
129
|
+
const response = parseResult.data;
|
130
|
+
if ("error" in response) {
|
131
|
+
queue.push({ type: "error", error: response });
|
132
|
+
return;
|
133
|
+
}
|
134
|
+
if (!response.isFinal) {
|
135
|
+
queue.push({
|
136
|
+
type: "delta",
|
137
|
+
fullDelta: event,
|
138
|
+
valueDelta: Buffer.from(response.audio, "base64"),
|
139
|
+
});
|
140
|
+
}
|
141
|
+
};
|
142
|
+
socket.onerror = (error) => {
|
143
|
+
queue.push({ type: "error", error });
|
144
|
+
};
|
145
|
+
socket.onclose = () => {
|
146
|
+
queue.close();
|
147
|
+
};
|
148
|
+
return queue;
|
149
|
+
}
|
150
|
+
withSettings(additionalSettings) {
|
151
|
+
return new ElevenLabsSpeechModel({
|
152
|
+
...this.settings,
|
153
|
+
...additionalSettings,
|
154
|
+
});
|
155
|
+
}
|
156
|
+
}
|
157
|
+
async function callElevenLabsTextToSpeechAPI({ api = new ElevenLabsApiConfiguration(), abortSignal, text, voiceId, modelId, voiceSettings, }) {
|
158
|
+
return postJsonToApi({
|
159
|
+
url: api.assembleUrl(`/text-to-speech/${voiceId}`),
|
160
|
+
headers: api.headers,
|
161
|
+
body: {
|
162
|
+
text,
|
163
|
+
model_id: modelId ?? defaultModel,
|
164
|
+
voice_settings: toApiVoiceSettings(voiceSettings),
|
165
|
+
},
|
166
|
+
failedResponseHandler: failedElevenLabsCallResponseHandler,
|
167
|
+
successfulResponseHandler: createAudioMpegResponseHandler(),
|
168
|
+
abortSignal,
|
169
|
+
});
|
170
|
+
}
|
171
|
+
function toApiVoiceSettings(voiceSettings) {
|
172
|
+
return voiceSettings != null
|
173
|
+
? {
|
174
|
+
stability: voiceSettings.stability,
|
175
|
+
similarity_boost: voiceSettings.similarityBoost,
|
176
|
+
style: voiceSettings.style,
|
177
|
+
use_speaker_boost: voiceSettings.useSpeakerBoost,
|
178
|
+
}
|
179
|
+
: undefined;
|
180
|
+
}
|
181
|
+
function toGenerationConfig(generationConfig) {
|
182
|
+
return generationConfig != null
|
183
|
+
? {
|
184
|
+
chunk_length_schedule: generationConfig.chunkLengthSchedule,
|
185
|
+
}
|
186
|
+
: undefined;
|
187
|
+
}
|
@@ -15,4 +15,4 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
15
15
|
};
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
17
17
|
__exportStar(require("./ElevenLabsApiConfiguration.cjs"), exports);
|
18
|
-
__exportStar(require("./
|
18
|
+
__exportStar(require("./ElevenLabsSpeechModel.cjs"), exports);
|
@@ -1,2 +1,2 @@
|
|
1
1
|
export * from "./ElevenLabsApiConfiguration.js";
|
2
|
-
export * from "./
|
2
|
+
export * from "./ElevenLabsSpeechModel.js";
|
@@ -1,2 +1,2 @@
|
|
1
1
|
export * from "./ElevenLabsApiConfiguration.js";
|
2
|
-
export * from "./
|
2
|
+
export * from "./ElevenLabsSpeechModel.js";
|
@@ -5,6 +5,7 @@ const zod_1 = require("zod");
|
|
5
5
|
const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
|
6
6
|
const postToApi_js_1 = require("../../core/api/postToApi.cjs");
|
7
7
|
const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
|
8
|
+
const PromptFormatTextGenerationModel_js_1 = require("../../model-function/generate-text/PromptFormatTextGenerationModel.cjs");
|
8
9
|
const HuggingFaceApiConfiguration_js_1 = require("./HuggingFaceApiConfiguration.cjs");
|
9
10
|
const HuggingFaceError_js_1 = require("./HuggingFaceError.cjs");
|
10
11
|
/**
|
@@ -21,6 +22,18 @@ class HuggingFaceImageDescriptionModel extends AbstractModel_js_1.AbstractModel
|
|
21
22
|
writable: true,
|
22
23
|
value: "huggingface"
|
23
24
|
});
|
25
|
+
Object.defineProperty(this, "contextWindowSize", {
|
26
|
+
enumerable: true,
|
27
|
+
configurable: true,
|
28
|
+
writable: true,
|
29
|
+
value: undefined
|
30
|
+
});
|
31
|
+
Object.defineProperty(this, "tokenizer", {
|
32
|
+
enumerable: true,
|
33
|
+
configurable: true,
|
34
|
+
writable: true,
|
35
|
+
value: undefined
|
36
|
+
});
|
24
37
|
Object.defineProperty(this, "countPromptTokens", {
|
25
38
|
enumerable: true,
|
26
39
|
configurable: true,
|
@@ -45,13 +58,19 @@ class HuggingFaceImageDescriptionModel extends AbstractModel_js_1.AbstractModel
|
|
45
58
|
get settingsForEvent() {
|
46
59
|
return {};
|
47
60
|
}
|
48
|
-
async
|
61
|
+
async doGenerateText(data, options) {
|
49
62
|
const response = await this.callAPI(data, options);
|
50
63
|
return {
|
51
64
|
response,
|
52
|
-
|
65
|
+
text: response[0].generated_text,
|
53
66
|
};
|
54
67
|
}
|
68
|
+
withPromptFormat(promptFormat) {
|
69
|
+
return new PromptFormatTextGenerationModel_js_1.PromptFormatTextGenerationModel({
|
70
|
+
model: this,
|
71
|
+
promptFormat,
|
72
|
+
});
|
73
|
+
}
|
55
74
|
withSettings(additionalSettings) {
|
56
75
|
return new HuggingFaceImageDescriptionModel(Object.assign({}, this.settings, additionalSettings));
|
57
76
|
}
|
@@ -1,10 +1,12 @@
|
|
1
1
|
/// <reference types="node" />
|
2
2
|
import { z } from "zod";
|
3
|
+
import { FunctionOptions } from "../../core/FunctionOptions.js";
|
3
4
|
import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
|
4
5
|
import { AbstractModel } from "../../model-function/AbstractModel.js";
|
5
|
-
import {
|
6
|
-
import {
|
7
|
-
|
6
|
+
import { PromptFormatTextGenerationModel } from "../../model-function/generate-text/PromptFormatTextGenerationModel.js";
|
7
|
+
import { TextGenerationModel, TextGenerationModelSettings } from "../../model-function/generate-text/TextGenerationModel.js";
|
8
|
+
import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
|
9
|
+
export interface HuggingFaceImageDescriptionModelSettings extends TextGenerationModelSettings {
|
8
10
|
api?: ApiConfiguration;
|
9
11
|
model: string;
|
10
12
|
}
|
@@ -13,19 +15,22 @@ export interface HuggingFaceImageDescriptionModelSettings extends ImageDescripti
|
|
13
15
|
*
|
14
16
|
* @see https://huggingface.co/tasks/image-to-text
|
15
17
|
*/
|
16
|
-
export declare class HuggingFaceImageDescriptionModel extends AbstractModel<HuggingFaceImageDescriptionModelSettings> implements
|
18
|
+
export declare class HuggingFaceImageDescriptionModel extends AbstractModel<HuggingFaceImageDescriptionModelSettings> implements TextGenerationModel<Buffer, HuggingFaceImageDescriptionModelSettings> {
|
17
19
|
constructor(settings: HuggingFaceImageDescriptionModelSettings);
|
18
20
|
readonly provider = "huggingface";
|
19
21
|
get modelName(): string;
|
20
22
|
callAPI(data: Buffer, options?: FunctionOptions): Promise<HuggingFaceImageDescriptionResponse>;
|
21
23
|
get settingsForEvent(): Partial<HuggingFaceImageDescriptionModelSettings>;
|
24
|
+
readonly contextWindowSize: undefined;
|
25
|
+
readonly tokenizer: undefined;
|
22
26
|
readonly countPromptTokens: undefined;
|
23
|
-
|
27
|
+
doGenerateText(data: Buffer, options?: FunctionOptions): Promise<{
|
24
28
|
response: {
|
25
29
|
generated_text: string;
|
26
30
|
}[];
|
27
|
-
|
31
|
+
text: string;
|
28
32
|
}>;
|
33
|
+
withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, Buffer>): PromptFormatTextGenerationModel<INPUT_PROMPT, Buffer, HuggingFaceImageDescriptionModelSettings, this>;
|
29
34
|
withSettings(additionalSettings: Partial<HuggingFaceImageDescriptionModelSettings>): this;
|
30
35
|
}
|
31
36
|
declare const huggingFaceImageDescriptionResponseSchema: z.ZodArray<z.ZodObject<{
|
@@ -2,6 +2,7 @@ import { z } from "zod";
|
|
2
2
|
import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
|
3
3
|
import { createJsonResponseHandler, postToApi, } from "../../core/api/postToApi.js";
|
4
4
|
import { AbstractModel } from "../../model-function/AbstractModel.js";
|
5
|
+
import { PromptFormatTextGenerationModel } from "../../model-function/generate-text/PromptFormatTextGenerationModel.js";
|
5
6
|
import { HuggingFaceApiConfiguration } from "./HuggingFaceApiConfiguration.js";
|
6
7
|
import { failedHuggingFaceCallResponseHandler } from "./HuggingFaceError.js";
|
7
8
|
/**
|
@@ -18,6 +19,18 @@ export class HuggingFaceImageDescriptionModel extends AbstractModel {
|
|
18
19
|
writable: true,
|
19
20
|
value: "huggingface"
|
20
21
|
});
|
22
|
+
Object.defineProperty(this, "contextWindowSize", {
|
23
|
+
enumerable: true,
|
24
|
+
configurable: true,
|
25
|
+
writable: true,
|
26
|
+
value: undefined
|
27
|
+
});
|
28
|
+
Object.defineProperty(this, "tokenizer", {
|
29
|
+
enumerable: true,
|
30
|
+
configurable: true,
|
31
|
+
writable: true,
|
32
|
+
value: undefined
|
33
|
+
});
|
21
34
|
Object.defineProperty(this, "countPromptTokens", {
|
22
35
|
enumerable: true,
|
23
36
|
configurable: true,
|
@@ -42,13 +55,19 @@ export class HuggingFaceImageDescriptionModel extends AbstractModel {
|
|
42
55
|
get settingsForEvent() {
|
43
56
|
return {};
|
44
57
|
}
|
45
|
-
async
|
58
|
+
async doGenerateText(data, options) {
|
46
59
|
const response = await this.callAPI(data, options);
|
47
60
|
return {
|
48
61
|
response,
|
49
|
-
|
62
|
+
text: response[0].generated_text,
|
50
63
|
};
|
51
64
|
}
|
65
|
+
withPromptFormat(promptFormat) {
|
66
|
+
return new PromptFormatTextGenerationModel({
|
67
|
+
model: this,
|
68
|
+
promptFormat,
|
69
|
+
});
|
70
|
+
}
|
52
71
|
withSettings(additionalSettings) {
|
53
72
|
return new HuggingFaceImageDescriptionModel(Object.assign({}, this.settings, additionalSettings));
|
54
73
|
}
|
@@ -1,6 +1,6 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.
|
3
|
+
exports.LmntSpeechModel = void 0;
|
4
4
|
const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
|
5
5
|
const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
|
6
6
|
const postToApi_js_1 = require("../../core/api/postToApi.cjs");
|
@@ -11,7 +11,7 @@ const LmntError_js_1 = require("./LmntError.cjs");
|
|
11
11
|
*
|
12
12
|
* @see https://www.lmnt.com/docs/rest/#synthesize-speech
|
13
13
|
*/
|
14
|
-
class
|
14
|
+
class LmntSpeechModel extends AbstractModel_js_1.AbstractModel {
|
15
15
|
constructor(settings) {
|
16
16
|
super({ settings });
|
17
17
|
Object.defineProperty(this, "provider", {
|
@@ -43,17 +43,17 @@ class LmntSpeechSynthesisModel extends AbstractModel_js_1.AbstractModel {
|
|
43
43
|
length: this.settings.length,
|
44
44
|
};
|
45
45
|
}
|
46
|
-
|
46
|
+
doGenerateSpeechStandard(text, options) {
|
47
47
|
return this.callAPI(text, options);
|
48
48
|
}
|
49
49
|
withSettings(additionalSettings) {
|
50
|
-
return new
|
50
|
+
return new LmntSpeechModel({
|
51
51
|
...this.settings,
|
52
52
|
...additionalSettings,
|
53
53
|
});
|
54
54
|
}
|
55
55
|
}
|
56
|
-
exports.
|
56
|
+
exports.LmntSpeechModel = LmntSpeechModel;
|
57
57
|
async function callLmntTextToSpeechAPI({ api = new LmntApiConfiguration_js_1.LmntApiConfiguration(), abortSignal, text, voice, speed, seed, length, }) {
|
58
58
|
const formData = new FormData();
|
59
59
|
formData.append("text", text);
|
@@ -0,0 +1,26 @@
|
|
1
|
+
/// <reference types="node" />
|
2
|
+
import { AbstractModel } from "../../model-function/AbstractModel.js";
|
3
|
+
import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
|
4
|
+
import { FunctionOptions } from "../../core/FunctionOptions.js";
|
5
|
+
import { SpeechGenerationModel, SpeechGenerationModelSettings } from "../../model-function/generate-speech/SpeechGenerationModel.js";
|
6
|
+
export interface LmntSpeechModelSettings extends SpeechGenerationModelSettings {
|
7
|
+
api?: ApiConfiguration;
|
8
|
+
voice: string;
|
9
|
+
speed?: number;
|
10
|
+
seed?: number;
|
11
|
+
length?: number;
|
12
|
+
}
|
13
|
+
/**
|
14
|
+
* Synthesize speech using the LMNT API.
|
15
|
+
*
|
16
|
+
* @see https://www.lmnt.com/docs/rest/#synthesize-speech
|
17
|
+
*/
|
18
|
+
export declare class LmntSpeechModel extends AbstractModel<LmntSpeechModelSettings> implements SpeechGenerationModel<LmntSpeechModelSettings> {
|
19
|
+
constructor(settings: LmntSpeechModelSettings);
|
20
|
+
readonly provider = "lmnt";
|
21
|
+
get modelName(): string;
|
22
|
+
private callAPI;
|
23
|
+
get settingsForEvent(): Partial<LmntSpeechModelSettings>;
|
24
|
+
doGenerateSpeechStandard(text: string, options?: FunctionOptions): Promise<Buffer>;
|
25
|
+
withSettings(additionalSettings: Partial<LmntSpeechModelSettings>): this;
|
26
|
+
}
|
@@ -8,7 +8,7 @@ import { failedLmntCallResponseHandler } from "./LmntError.js";
|
|
8
8
|
*
|
9
9
|
* @see https://www.lmnt.com/docs/rest/#synthesize-speech
|
10
10
|
*/
|
11
|
-
export class
|
11
|
+
export class LmntSpeechModel extends AbstractModel {
|
12
12
|
constructor(settings) {
|
13
13
|
super({ settings });
|
14
14
|
Object.defineProperty(this, "provider", {
|
@@ -40,11 +40,11 @@ export class LmntSpeechSynthesisModel extends AbstractModel {
|
|
40
40
|
length: this.settings.length,
|
41
41
|
};
|
42
42
|
}
|
43
|
-
|
43
|
+
doGenerateSpeechStandard(text, options) {
|
44
44
|
return this.callAPI(text, options);
|
45
45
|
}
|
46
46
|
withSettings(additionalSettings) {
|
47
|
-
return new
|
47
|
+
return new LmntSpeechModel({
|
48
48
|
...this.settings,
|
49
49
|
...additionalSettings,
|
50
50
|
});
|
@@ -15,4 +15,4 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
15
15
|
};
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
17
17
|
__exportStar(require("./LmntApiConfiguration.cjs"), exports);
|
18
|
-
__exportStar(require("./
|
18
|
+
__exportStar(require("./LmntSpeechModel.cjs"), exports);
|
@@ -1,2 +1,2 @@
|
|
1
1
|
export * from "./LmntApiConfiguration.js";
|
2
|
-
export * from "./
|
2
|
+
export * from "./LmntSpeechModel.js";
|
@@ -1,2 +1,2 @@
|
|
1
1
|
export * from "./LmntApiConfiguration.js";
|
2
|
-
export * from "./
|
2
|
+
export * from "./LmntSpeechModel.js";
|
@@ -1,6 +1,6 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.OpenAITextResponseFormat = exports.
|
3
|
+
exports.OpenAITextResponseFormat = exports.OpenAICompletionModel = exports.calculateOpenAICompletionCostInMillicents = exports.isOpenAICompletionModel = exports.getOpenAICompletionModelInformation = exports.OPENAI_TEXT_GENERATION_MODELS = void 0;
|
4
4
|
const zod_1 = require("zod");
|
5
5
|
const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
|
6
6
|
const postToApi_js_1 = require("../../core/api/postToApi.cjs");
|
@@ -87,7 +87,7 @@ exports.OPENAI_TEXT_GENERATION_MODELS = {
|
|
87
87
|
completionTokenCostInMillicents: 0.04,
|
88
88
|
},
|
89
89
|
};
|
90
|
-
function
|
90
|
+
function getOpenAICompletionModelInformation(model) {
|
91
91
|
// Model is already a base model:
|
92
92
|
if (model in exports.OPENAI_TEXT_GENERATION_MODELS) {
|
93
93
|
const baseModelInformation = exports.OPENAI_TEXT_GENERATION_MODELS[model];
|
@@ -114,26 +114,26 @@ function getOpenAITextGenerationModelInformation(model) {
|
|
114
114
|
}
|
115
115
|
throw new Error(`Unknown OpenAI chat base model ${baseModel}.`);
|
116
116
|
}
|
117
|
-
exports.
|
118
|
-
const
|
117
|
+
exports.getOpenAICompletionModelInformation = getOpenAICompletionModelInformation;
|
118
|
+
const isOpenAICompletionModel = (model) => model in exports.OPENAI_TEXT_GENERATION_MODELS ||
|
119
119
|
model.startsWith("ft:davinci-002:") ||
|
120
120
|
model.startsWith("ft:babbage-002:");
|
121
|
-
exports.
|
122
|
-
const
|
123
|
-
const modelInformation =
|
121
|
+
exports.isOpenAICompletionModel = isOpenAICompletionModel;
|
122
|
+
const calculateOpenAICompletionCostInMillicents = ({ model, response, }) => {
|
123
|
+
const modelInformation = getOpenAICompletionModelInformation(model);
|
124
124
|
return (response.usage.prompt_tokens *
|
125
125
|
modelInformation.promptTokenCostInMillicents +
|
126
126
|
response.usage.completion_tokens *
|
127
127
|
modelInformation.completionTokenCostInMillicents);
|
128
128
|
};
|
129
|
-
exports.
|
129
|
+
exports.calculateOpenAICompletionCostInMillicents = calculateOpenAICompletionCostInMillicents;
|
130
130
|
/**
|
131
131
|
* Create a text generation model that calls the OpenAI text completion API.
|
132
132
|
*
|
133
133
|
* @see https://platform.openai.com/docs/api-reference/completions/create
|
134
134
|
*
|
135
135
|
* @example
|
136
|
-
* const model = new
|
136
|
+
* const model = new OpenAICompletionModel({
|
137
137
|
* model: "gpt-3.5-turbo-instruct",
|
138
138
|
* temperature: 0.7,
|
139
139
|
* maxCompletionTokens: 500,
|
@@ -145,7 +145,7 @@ exports.calculateOpenAITextGenerationCostInMillicents = calculateOpenAITextGener
|
|
145
145
|
* "Write a short story about a robot learning to love:\n\n"
|
146
146
|
* );
|
147
147
|
*/
|
148
|
-
class
|
148
|
+
class OpenAICompletionModel extends AbstractModel_js_1.AbstractModel {
|
149
149
|
constructor(settings) {
|
150
150
|
super({ settings });
|
151
151
|
Object.defineProperty(this, "provider", {
|
@@ -166,7 +166,7 @@ class OpenAITextGenerationModel extends AbstractModel_js_1.AbstractModel {
|
|
166
166
|
writable: true,
|
167
167
|
value: void 0
|
168
168
|
});
|
169
|
-
const modelInformation =
|
169
|
+
const modelInformation = getOpenAICompletionModelInformation(this.settings.model);
|
170
170
|
this.tokenizer = new TikTokenTokenizer_js_1.TikTokenTokenizer({
|
171
171
|
model: modelInformation.baseModel,
|
172
172
|
});
|
@@ -195,7 +195,7 @@ class OpenAITextGenerationModel extends AbstractModel_js_1.AbstractModel {
|
|
195
195
|
return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
|
196
196
|
retry: callSettings.api?.retry,
|
197
197
|
throttle: callSettings.api?.throttle,
|
198
|
-
call: async () =>
|
198
|
+
call: async () => callOpenAICompletionAPI(callSettings),
|
199
199
|
});
|
200
200
|
}
|
201
201
|
get settingsForEvent() {
|
@@ -260,11 +260,11 @@ class OpenAITextGenerationModel extends AbstractModel_js_1.AbstractModel {
|
|
260
260
|
});
|
261
261
|
}
|
262
262
|
withSettings(additionalSettings) {
|
263
|
-
return new
|
263
|
+
return new OpenAICompletionModel(Object.assign({}, this.settings, additionalSettings));
|
264
264
|
}
|
265
265
|
}
|
266
|
-
exports.
|
267
|
-
const
|
266
|
+
exports.OpenAICompletionModel = OpenAICompletionModel;
|
267
|
+
const OpenAICompletionResponseSchema = zod_1.z.object({
|
268
268
|
id: zod_1.z.string(),
|
269
269
|
object: zod_1.z.literal("text_completion"),
|
270
270
|
created: zod_1.z.number(),
|
@@ -281,7 +281,7 @@ const openAITextGenerationResponseSchema = zod_1.z.object({
|
|
281
281
|
total_tokens: zod_1.z.number(),
|
282
282
|
}),
|
283
283
|
});
|
284
|
-
async function
|
284
|
+
async function callOpenAICompletionAPI({ api = new OpenAIApiConfiguration_js_1.OpenAIApiConfiguration(), abortSignal, responseFormat, model, prompt, suffix, maxTokens, temperature, topP, n, logprobs, echo, stop, presencePenalty, frequencyPenalty, bestOf, logitBias, user, }) {
|
285
285
|
// empty arrays are not allowed for stop:
|
286
286
|
if (stop != null && Array.isArray(stop) && stop.length === 0) {
|
287
287
|
stop = undefined;
|
@@ -318,7 +318,7 @@ exports.OpenAITextResponseFormat = {
|
|
318
318
|
*/
|
319
319
|
json: {
|
320
320
|
stream: false,
|
321
|
-
handler: (0, postToApi_js_1.createJsonResponseHandler)(
|
321
|
+
handler: (0, postToApi_js_1.createJsonResponseHandler)(OpenAICompletionResponseSchema),
|
322
322
|
},
|
323
323
|
/**
|
324
324
|
* Returns an async iterable over the full deltas (all choices, including full current state at time of event)
|