modelfusion 0.53.1 → 0.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/browser/convertAudioChunksToBase64.cjs +8 -0
- package/browser/convertAudioChunksToBase64.d.ts +4 -0
- package/browser/convertAudioChunksToBase64.js +4 -0
- package/browser/convertBlobToBase64.cjs +23 -0
- package/browser/convertBlobToBase64.d.ts +1 -0
- package/browser/convertBlobToBase64.js +19 -0
- package/{ui → browser}/index.cjs +5 -0
- package/browser/index.d.ts +6 -0
- package/browser/index.js +6 -0
- package/browser/invokeFlow.cjs +23 -0
- package/browser/invokeFlow.d.ts +8 -0
- package/browser/invokeFlow.js +19 -0
- package/{event-source → browser}/readEventSource.cjs +8 -3
- package/{event-source → browser}/readEventSource.d.ts +3 -1
- package/{event-source → browser}/readEventSource.js +8 -3
- package/{event-source → browser}/readEventSourceStream.cjs +1 -1
- package/{event-source → browser}/readEventSourceStream.js +1 -1
- package/core/FunctionOptions.d.ts +4 -3
- package/event-source/createEventSourceStream.cjs +7 -3
- package/event-source/createEventSourceStream.js +7 -3
- package/event-source/index.cjs +0 -2
- package/event-source/index.d.ts +0 -2
- package/event-source/index.js +0 -2
- package/index.cjs +0 -1
- package/index.d.ts +0 -1
- package/index.js +0 -1
- package/model-function/embed/embed.cjs +16 -0
- package/model-function/embed/embed.d.ts +16 -0
- package/model-function/embed/embed.js +16 -0
- package/model-function/generate-image/generateImage.cjs +12 -3
- package/model-function/generate-image/generateImage.d.ts +12 -3
- package/model-function/generate-image/generateImage.js +12 -3
- package/model-function/generate-speech/generateSpeech.cjs +16 -1
- package/model-function/generate-speech/generateSpeech.d.ts +16 -1
- package/model-function/generate-speech/generateSpeech.js +16 -1
- package/model-function/generate-speech/streamSpeech.cjs +22 -1
- package/model-function/generate-speech/streamSpeech.d.ts +22 -1
- package/model-function/generate-speech/streamSpeech.js +22 -1
- package/model-function/generate-structure/generateStructure.cjs +41 -0
- package/model-function/generate-structure/generateStructure.d.ts +41 -0
- package/model-function/generate-structure/generateStructure.js +41 -0
- package/model-function/generate-structure/generateStructureOrText.cjs +62 -0
- package/model-function/generate-structure/generateStructureOrText.d.ts +62 -0
- package/model-function/generate-structure/generateStructureOrText.js +62 -0
- package/model-function/generate-structure/streamStructure.cjs +72 -1
- package/model-function/generate-structure/streamStructure.d.ts +68 -1
- package/model-function/generate-structure/streamStructure.js +72 -1
- package/model-function/generate-text/generateText.cjs +14 -6
- package/model-function/generate-text/generateText.d.ts +14 -6
- package/model-function/generate-text/generateText.js +14 -6
- package/model-function/generate-text/streamText.cjs +25 -0
- package/model-function/generate-text/streamText.d.ts +25 -0
- package/model-function/generate-text/streamText.js +25 -0
- package/model-function/generate-transcription/generateTranscription.cjs +10 -5
- package/model-function/generate-transcription/generateTranscription.d.ts +10 -5
- package/model-function/generate-transcription/generateTranscription.js +10 -5
- package/model-function/tokenize-text/Tokenizer.d.ts +27 -3
- package/package.json +13 -4
- package/server/fastify/AssetStorage.cjs +2 -0
- package/server/fastify/AssetStorage.d.ts +17 -0
- package/server/fastify/AssetStorage.js +1 -0
- package/server/fastify/DefaultFlow.cjs +22 -0
- package/server/fastify/DefaultFlow.d.ts +16 -0
- package/server/fastify/DefaultFlow.js +18 -0
- package/server/fastify/FileSystemAssetStorage.cjs +60 -0
- package/server/fastify/FileSystemAssetStorage.d.ts +19 -0
- package/server/fastify/FileSystemAssetStorage.js +56 -0
- package/server/fastify/FileSystemLogger.cjs +48 -0
- package/server/fastify/FileSystemLogger.d.ts +18 -0
- package/server/fastify/FileSystemLogger.js +44 -0
- package/server/fastify/Flow.cjs +2 -0
- package/server/fastify/Flow.d.ts +9 -0
- package/server/fastify/Flow.js +1 -0
- package/server/fastify/FlowRun.cjs +71 -0
- package/server/fastify/FlowRun.d.ts +28 -0
- package/server/fastify/FlowRun.js +67 -0
- package/server/fastify/FlowSchema.cjs +2 -0
- package/server/fastify/FlowSchema.d.ts +5 -0
- package/server/fastify/FlowSchema.js +1 -0
- package/server/fastify/Logger.cjs +2 -0
- package/server/fastify/Logger.d.ts +13 -0
- package/server/fastify/Logger.js +1 -0
- package/server/fastify/PathProvider.cjs +34 -0
- package/server/fastify/PathProvider.d.ts +12 -0
- package/server/fastify/PathProvider.js +30 -0
- package/server/fastify/index.cjs +24 -0
- package/server/fastify/index.d.ts +8 -0
- package/server/fastify/index.js +8 -0
- package/server/fastify/modelFusionFlowPlugin.cjs +102 -0
- package/server/fastify/modelFusionFlowPlugin.d.ts +12 -0
- package/server/fastify/modelFusionFlowPlugin.js +98 -0
- package/ui/index.d.ts +0 -1
- package/ui/index.js +0 -1
- /package/{ui → browser}/MediaSourceAppender.cjs +0 -0
- /package/{ui → browser}/MediaSourceAppender.d.ts +0 -0
- /package/{ui → browser}/MediaSourceAppender.js +0 -0
- /package/{event-source → browser}/readEventSourceStream.d.ts +0 -0
@@ -1,7 +1,78 @@
|
|
1
1
|
import { isDeepEqualData } from "../../util/isDeepEqualData.js";
|
2
2
|
import { AsyncIterableResultPromise } from "../AsyncIterableResultPromise.js";
|
3
3
|
import { executeStreamCall } from "../executeStreamCall.js";
|
4
|
+
/**
|
5
|
+
* Generate and stream an object for a prompt and a structure definition.
|
6
|
+
*
|
7
|
+
* The final object is typed according to the structure definition.
|
8
|
+
* The partial objects are of unknown type,
|
9
|
+
* but are supposed to be partial version of the final object
|
10
|
+
* (unless the underlying model returns invalid data).
|
11
|
+
*
|
12
|
+
* The structure definition is used as part of the final prompt.
|
13
|
+
*
|
14
|
+
* For the OpenAI chat model, this generates and parses a function call with a single function.
|
15
|
+
*
|
16
|
+
* @see https://modelfusion.dev/guide/function/generate-structure
|
17
|
+
*
|
18
|
+
* @example
|
19
|
+
* const structureStream = await streamStructure(
|
20
|
+
* new OpenAIChatModel({
|
21
|
+
* model: "gpt-3.5-turbo",
|
22
|
+
* temperature: 0,
|
23
|
+
* maxCompletionTokens: 2000,
|
24
|
+
* }),
|
25
|
+
* new ZodStructureDefinition({
|
26
|
+
* name: "generateCharacter",
|
27
|
+
* description: "Generate character descriptions.",
|
28
|
+
* schema: z.object({
|
29
|
+
* characters: z.array(
|
30
|
+
* z.object({
|
31
|
+
* name: z.string(),
|
32
|
+
* class: z
|
33
|
+
* .string()
|
34
|
+
* .describe("Character class, e.g. warrior, mage, or thief."),
|
35
|
+
* description: z.string(),
|
36
|
+
* })
|
37
|
+
* ),
|
38
|
+
* }),
|
39
|
+
* }),
|
40
|
+
* [
|
41
|
+
* OpenAIChatMessage.user(
|
42
|
+
* "Generate 3 character descriptions for a fantasy role playing game."
|
43
|
+
* ),
|
44
|
+
* ]
|
45
|
+
* );
|
46
|
+
*
|
47
|
+
* for await (const part of structureStream) {
|
48
|
+
* if (!part.isComplete) {
|
49
|
+
* const unknownPartialStructure = part.value;
|
50
|
+
* // use your own logic to handle partial structures, e.g. with Zod .deepPartial()
|
51
|
+
* // it depends on your application at which points you want to act on the partial structures
|
52
|
+
* } else {
|
53
|
+
* const fullyTypedStructure = part.value;
|
54
|
+
* // ...
|
55
|
+
* }
|
56
|
+
* }
|
57
|
+
*
|
58
|
+
* @param {StructureStreamingModel<PROMPT>} model - The model to use for streaming
|
59
|
+
* @param {StructureDefinition<NAME, STRUCTURE>} structureDefinition - The structure definition to use
|
60
|
+
* @param {PROMPT | ((structureDefinition: StructureDefinition<NAME, STRUCTURE>) => PROMPT)} prompt
|
61
|
+
* The prompt to be used.
|
62
|
+
* You can also pass a function that takes the structure definition as an argument and returns the prompt.
|
63
|
+
* @param {FunctionOptions} [options] - Optional function options
|
64
|
+
*
|
65
|
+
* @returns {AsyncIterableResultPromise<StructureStreamPart<STRUCTURE>>}
|
66
|
+
* The async iterable result promise.
|
67
|
+
* Each part of the stream is either a partial structure or the final structure.
|
68
|
+
* It contains a isComplete flag to indicate whether the structure is complete,
|
69
|
+
* and a value that is either the partial structure or the final structure.
|
70
|
+
*/
|
4
71
|
export function streamStructure(model, structureDefinition, prompt, options) {
|
72
|
+
// Note: PROMPT must not be a function.
|
73
|
+
const expandedPrompt = typeof prompt === "function"
|
74
|
+
? prompt(structureDefinition)
|
75
|
+
: prompt;
|
5
76
|
let lastStructure;
|
6
77
|
let lastFullDelta;
|
7
78
|
return new AsyncIterableResultPromise(executeStreamCall({
|
@@ -9,7 +80,7 @@ export function streamStructure(model, structureDefinition, prompt, options) {
|
|
9
80
|
input: prompt,
|
10
81
|
model,
|
11
82
|
options,
|
12
|
-
startStream: async (options) => model.doStreamStructure(structureDefinition,
|
83
|
+
startStream: async (options) => model.doStreamStructure(structureDefinition, expandedPrompt, options),
|
13
84
|
processDelta: (delta) => {
|
14
85
|
const latestFullDelta = delta.fullDelta;
|
15
86
|
const latestStructure = delta.valueDelta;
|
@@ -4,17 +4,25 @@ exports.generateText = void 0;
|
|
4
4
|
const executeStandardCall_js_1 = require("../executeStandardCall.cjs");
|
5
5
|
const ModelFunctionPromise_js_1 = require("../ModelFunctionPromise.cjs");
|
6
6
|
/**
|
7
|
-
*
|
8
|
-
* The prompt format depends on the model.
|
9
|
-
* For example, OpenAI text models expect a string prompt, and OpenAI chat models expect an array of chat messages.
|
7
|
+
* Generate text for a prompt and return it as a string.
|
10
8
|
*
|
11
|
-
*
|
12
|
-
*
|
9
|
+
* The prompt depends on the model used.
|
10
|
+
* For instance, OpenAI completion models expect a string prompt,
|
11
|
+
* whereas OpenAI chat models expect an array of chat messages.
|
12
|
+
*
|
13
|
+
* @see https://modelfusion.dev/guide/function/generate-text
|
13
14
|
*
|
15
|
+
* @example
|
14
16
|
* const text = await generateText(
|
15
|
-
*
|
17
|
+
* new OpenAICompletionModel(...),
|
16
18
|
* "Write a short story about a robot learning to love:\n\n"
|
17
19
|
* );
|
20
|
+
*
|
21
|
+
* @param {TextGenerationModel<PROMPT, TextGenerationModelSettings>} model - The text generation model to use.
|
22
|
+
* @param {PROMPT} prompt - The prompt to use for text generation.
|
23
|
+
* @param {FunctionOptions} [options] - Optional parameters for the function.
|
24
|
+
*
|
25
|
+
* @returns {ModelFunctionPromise<string>} - A promise that resolves to the generated text.
|
18
26
|
*/
|
19
27
|
function generateText(model, prompt, options) {
|
20
28
|
return new ModelFunctionPromise_js_1.ModelFunctionPromise((0, executeStandardCall_js_1.executeStandardCall)({
|
@@ -2,16 +2,24 @@ import { FunctionOptions } from "../../core/FunctionOptions.js";
|
|
2
2
|
import { ModelFunctionPromise } from "../ModelFunctionPromise.js";
|
3
3
|
import { TextGenerationModel, TextGenerationModelSettings } from "./TextGenerationModel.js";
|
4
4
|
/**
|
5
|
-
*
|
6
|
-
* The prompt format depends on the model.
|
7
|
-
* For example, OpenAI text models expect a string prompt, and OpenAI chat models expect an array of chat messages.
|
5
|
+
* Generate text for a prompt and return it as a string.
|
8
6
|
*
|
9
|
-
*
|
10
|
-
*
|
7
|
+
* The prompt depends on the model used.
|
8
|
+
* For instance, OpenAI completion models expect a string prompt,
|
9
|
+
* whereas OpenAI chat models expect an array of chat messages.
|
10
|
+
*
|
11
|
+
* @see https://modelfusion.dev/guide/function/generate-text
|
11
12
|
*
|
13
|
+
* @example
|
12
14
|
* const text = await generateText(
|
13
|
-
*
|
15
|
+
* new OpenAICompletionModel(...),
|
14
16
|
* "Write a short story about a robot learning to love:\n\n"
|
15
17
|
* );
|
18
|
+
*
|
19
|
+
* @param {TextGenerationModel<PROMPT, TextGenerationModelSettings>} model - The text generation model to use.
|
20
|
+
* @param {PROMPT} prompt - The prompt to use for text generation.
|
21
|
+
* @param {FunctionOptions} [options] - Optional parameters for the function.
|
22
|
+
*
|
23
|
+
* @returns {ModelFunctionPromise<string>} - A promise that resolves to the generated text.
|
16
24
|
*/
|
17
25
|
export declare function generateText<PROMPT>(model: TextGenerationModel<PROMPT, TextGenerationModelSettings>, prompt: PROMPT, options?: FunctionOptions): ModelFunctionPromise<string>;
|
@@ -1,17 +1,25 @@
|
|
1
1
|
import { executeStandardCall } from "../executeStandardCall.js";
|
2
2
|
import { ModelFunctionPromise } from "../ModelFunctionPromise.js";
|
3
3
|
/**
|
4
|
-
*
|
5
|
-
* The prompt format depends on the model.
|
6
|
-
* For example, OpenAI text models expect a string prompt, and OpenAI chat models expect an array of chat messages.
|
4
|
+
* Generate text for a prompt and return it as a string.
|
7
5
|
*
|
8
|
-
*
|
9
|
-
*
|
6
|
+
* The prompt depends on the model used.
|
7
|
+
* For instance, OpenAI completion models expect a string prompt,
|
8
|
+
* whereas OpenAI chat models expect an array of chat messages.
|
9
|
+
*
|
10
|
+
* @see https://modelfusion.dev/guide/function/generate-text
|
10
11
|
*
|
12
|
+
* @example
|
11
13
|
* const text = await generateText(
|
12
|
-
*
|
14
|
+
* new OpenAICompletionModel(...),
|
13
15
|
* "Write a short story about a robot learning to love:\n\n"
|
14
16
|
* );
|
17
|
+
*
|
18
|
+
* @param {TextGenerationModel<PROMPT, TextGenerationModelSettings>} model - The text generation model to use.
|
19
|
+
* @param {PROMPT} prompt - The prompt to use for text generation.
|
20
|
+
* @param {FunctionOptions} [options] - Optional parameters for the function.
|
21
|
+
*
|
22
|
+
* @returns {ModelFunctionPromise<string>} - A promise that resolves to the generated text.
|
15
23
|
*/
|
16
24
|
export function generateText(model, prompt, options) {
|
17
25
|
return new ModelFunctionPromise(executeStandardCall({
|
@@ -3,6 +3,31 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.streamText = void 0;
|
4
4
|
const AsyncIterableResultPromise_js_1 = require("../AsyncIterableResultPromise.cjs");
|
5
5
|
const executeStreamCall_js_1 = require("../executeStreamCall.cjs");
|
6
|
+
/**
|
7
|
+
* Stream the generated text for a prompt as an async iterable.
|
8
|
+
*
|
9
|
+
* The prompt depends on the model used.
|
10
|
+
* For instance, OpenAI completion models expect a string prompt,
|
11
|
+
* whereas OpenAI chat models expect an array of chat messages.
|
12
|
+
*
|
13
|
+
* @see https://modelfusion.dev/guide/function/generate-text
|
14
|
+
*
|
15
|
+
* @example
|
16
|
+
* const textStream = await streamText(
|
17
|
+
* new OpenAICompletionModel(...),
|
18
|
+
* "Write a short story about a robot learning to love:\n\n"
|
19
|
+
* );
|
20
|
+
*
|
21
|
+
* for await (const textPart of textStream) {
|
22
|
+
* // ...
|
23
|
+
* }
|
24
|
+
*
|
25
|
+
* @param {TextStreamingModel<PROMPT>} model - The model to stream text from.
|
26
|
+
* @param {PROMPT} prompt - The prompt to use for text generation.
|
27
|
+
* @param {FunctionOptions} [options] - Optional parameters for the function.
|
28
|
+
*
|
29
|
+
* @returns {AsyncIterableResultPromise<string>} An async iterable promise that yields the generated text.
|
30
|
+
*/
|
6
31
|
function streamText(model, prompt, options) {
|
7
32
|
let accumulatedText = "";
|
8
33
|
let lastFullDelta;
|
@@ -1,4 +1,29 @@
|
|
1
1
|
import { FunctionOptions } from "../../core/FunctionOptions.js";
|
2
2
|
import { AsyncIterableResultPromise } from "../AsyncIterableResultPromise.js";
|
3
3
|
import { TextStreamingModel } from "./TextGenerationModel.js";
|
4
|
+
/**
|
5
|
+
* Stream the generated text for a prompt as an async iterable.
|
6
|
+
*
|
7
|
+
* The prompt depends on the model used.
|
8
|
+
* For instance, OpenAI completion models expect a string prompt,
|
9
|
+
* whereas OpenAI chat models expect an array of chat messages.
|
10
|
+
*
|
11
|
+
* @see https://modelfusion.dev/guide/function/generate-text
|
12
|
+
*
|
13
|
+
* @example
|
14
|
+
* const textStream = await streamText(
|
15
|
+
* new OpenAICompletionModel(...),
|
16
|
+
* "Write a short story about a robot learning to love:\n\n"
|
17
|
+
* );
|
18
|
+
*
|
19
|
+
* for await (const textPart of textStream) {
|
20
|
+
* // ...
|
21
|
+
* }
|
22
|
+
*
|
23
|
+
* @param {TextStreamingModel<PROMPT>} model - The model to stream text from.
|
24
|
+
* @param {PROMPT} prompt - The prompt to use for text generation.
|
25
|
+
* @param {FunctionOptions} [options] - Optional parameters for the function.
|
26
|
+
*
|
27
|
+
* @returns {AsyncIterableResultPromise<string>} An async iterable promise that yields the generated text.
|
28
|
+
*/
|
4
29
|
export declare function streamText<PROMPT>(model: TextStreamingModel<PROMPT>, prompt: PROMPT, options?: FunctionOptions): AsyncIterableResultPromise<string>;
|
@@ -1,5 +1,30 @@
|
|
1
1
|
import { AsyncIterableResultPromise } from "../AsyncIterableResultPromise.js";
|
2
2
|
import { executeStreamCall } from "../executeStreamCall.js";
|
3
|
+
/**
|
4
|
+
* Stream the generated text for a prompt as an async iterable.
|
5
|
+
*
|
6
|
+
* The prompt depends on the model used.
|
7
|
+
* For instance, OpenAI completion models expect a string prompt,
|
8
|
+
* whereas OpenAI chat models expect an array of chat messages.
|
9
|
+
*
|
10
|
+
* @see https://modelfusion.dev/guide/function/generate-text
|
11
|
+
*
|
12
|
+
* @example
|
13
|
+
* const textStream = await streamText(
|
14
|
+
* new OpenAICompletionModel(...),
|
15
|
+
* "Write a short story about a robot learning to love:\n\n"
|
16
|
+
* );
|
17
|
+
*
|
18
|
+
* for await (const textPart of textStream) {
|
19
|
+
* // ...
|
20
|
+
* }
|
21
|
+
*
|
22
|
+
* @param {TextStreamingModel<PROMPT>} model - The model to stream text from.
|
23
|
+
* @param {PROMPT} prompt - The prompt to use for text generation.
|
24
|
+
* @param {FunctionOptions} [options] - Optional parameters for the function.
|
25
|
+
*
|
26
|
+
* @returns {AsyncIterableResultPromise<string>} An async iterable promise that yields the generated text.
|
27
|
+
*/
|
3
28
|
export function streamText(model, prompt, options) {
|
4
29
|
let accumulatedText = "";
|
5
30
|
let lastFullDelta;
|
@@ -4,18 +4,23 @@ exports.generateTranscription = void 0;
|
|
4
4
|
const executeStandardCall_js_1 = require("../executeStandardCall.cjs");
|
5
5
|
const ModelFunctionPromise_js_1 = require("../ModelFunctionPromise.cjs");
|
6
6
|
/**
|
7
|
-
* Transcribe audio data into text.
|
7
|
+
* Transcribe audio data into text. Also called speech-to-text (STT) or automatic speech recognition (ASR).
|
8
|
+
*
|
9
|
+
* @see https://modelfusion.dev/guide/function/generate-transcription
|
8
10
|
*
|
9
11
|
* @example
|
10
12
|
* const data = await fs.promises.readFile("data/test.mp3");
|
11
13
|
*
|
12
14
|
* const transcription = await generateTranscription(
|
13
15
|
* new OpenAITranscriptionModel({ model: "whisper-1" }),
|
14
|
-
* {
|
15
|
-
* type: "mp3",
|
16
|
-
* data,
|
17
|
-
* }
|
16
|
+
* { type: "mp3", data }
|
18
17
|
* );
|
18
|
+
*
|
19
|
+
* @param {TranscriptionModel<DATA, TranscriptionModelSettings>} model - The model to use for transcription.
|
20
|
+
* @param {DATA} data - The data to transcribe.
|
21
|
+
* @param {FunctionOptions} [options] - Optional parameters for the function.
|
22
|
+
*
|
23
|
+
* @returns {ModelFunctionPromise<string>} A promise that resolves to the transcribed text.
|
19
24
|
*/
|
20
25
|
function generateTranscription(model, data, options) {
|
21
26
|
return new ModelFunctionPromise_js_1.ModelFunctionPromise((0, executeStandardCall_js_1.executeStandardCall)({
|
@@ -2,17 +2,22 @@ import { FunctionOptions } from "../../core/FunctionOptions.js";
|
|
2
2
|
import { ModelFunctionPromise } from "../ModelFunctionPromise.js";
|
3
3
|
import { TranscriptionModel, TranscriptionModelSettings } from "./TranscriptionModel.js";
|
4
4
|
/**
|
5
|
-
* Transcribe audio data into text.
|
5
|
+
* Transcribe audio data into text. Also called speech-to-text (STT) or automatic speech recognition (ASR).
|
6
|
+
*
|
7
|
+
* @see https://modelfusion.dev/guide/function/generate-transcription
|
6
8
|
*
|
7
9
|
* @example
|
8
10
|
* const data = await fs.promises.readFile("data/test.mp3");
|
9
11
|
*
|
10
12
|
* const transcription = await generateTranscription(
|
11
13
|
* new OpenAITranscriptionModel({ model: "whisper-1" }),
|
12
|
-
* {
|
13
|
-
* type: "mp3",
|
14
|
-
* data,
|
15
|
-
* }
|
14
|
+
* { type: "mp3", data }
|
16
15
|
* );
|
16
|
+
*
|
17
|
+
* @param {TranscriptionModel<DATA, TranscriptionModelSettings>} model - The model to use for transcription.
|
18
|
+
* @param {DATA} data - The data to transcribe.
|
19
|
+
* @param {FunctionOptions} [options] - Optional parameters for the function.
|
20
|
+
*
|
21
|
+
* @returns {ModelFunctionPromise<string>} A promise that resolves to the transcribed text.
|
17
22
|
*/
|
18
23
|
export declare function generateTranscription<DATA>(model: TranscriptionModel<DATA, TranscriptionModelSettings>, data: DATA, options?: FunctionOptions): ModelFunctionPromise<string>;
|
@@ -1,18 +1,23 @@
|
|
1
1
|
import { executeStandardCall } from "../executeStandardCall.js";
|
2
2
|
import { ModelFunctionPromise } from "../ModelFunctionPromise.js";
|
3
3
|
/**
|
4
|
-
* Transcribe audio data into text.
|
4
|
+
* Transcribe audio data into text. Also called speech-to-text (STT) or automatic speech recognition (ASR).
|
5
|
+
*
|
6
|
+
* @see https://modelfusion.dev/guide/function/generate-transcription
|
5
7
|
*
|
6
8
|
* @example
|
7
9
|
* const data = await fs.promises.readFile("data/test.mp3");
|
8
10
|
*
|
9
11
|
* const transcription = await generateTranscription(
|
10
12
|
* new OpenAITranscriptionModel({ model: "whisper-1" }),
|
11
|
-
* {
|
12
|
-
* type: "mp3",
|
13
|
-
* data,
|
14
|
-
* }
|
13
|
+
* { type: "mp3", data }
|
15
14
|
* );
|
15
|
+
*
|
16
|
+
* @param {TranscriptionModel<DATA, TranscriptionModelSettings>} model - The model to use for transcription.
|
17
|
+
* @param {DATA} data - The data to transcribe.
|
18
|
+
* @param {FunctionOptions} [options] - Optional parameters for the function.
|
19
|
+
*
|
20
|
+
* @returns {ModelFunctionPromise<string>} A promise that resolves to the transcribed text.
|
16
21
|
*/
|
17
22
|
export function generateTranscription(model, data, options) {
|
18
23
|
return new ModelFunctionPromise(executeStandardCall({
|
@@ -1,19 +1,43 @@
|
|
1
|
+
/**
|
2
|
+
* Interface for a basic tokenizer capable of converting text into tokens.
|
3
|
+
*
|
4
|
+
* This serves as the base for tokenization functionalities where the focus is on the transformation of input text into a series of numeric tokens.
|
5
|
+
*/
|
1
6
|
export interface BasicTokenizer {
|
2
7
|
/**
|
3
|
-
*
|
8
|
+
* Asynchronously tokenize the given text into a sequence of numeric tokens.
|
9
|
+
*
|
10
|
+
* @param text - Input text string that needs to be tokenized.
|
11
|
+
* @returns A promise containing an array of numbers, where each number is a token representing a part or the whole of the input text.
|
4
12
|
*/
|
5
13
|
tokenize: (text: string) => PromiseLike<Array<number>>;
|
6
14
|
}
|
15
|
+
/**
|
16
|
+
* Interface for a comprehensive tokenizer that extends the basic tokenization capabilities.
|
17
|
+
*
|
18
|
+
* In addition to basic tokenization, this interface provides methods for detokenization and
|
19
|
+
* retrieving the original text corresponding to each token, enabling a more informative and reversible transformation process.
|
20
|
+
*/
|
7
21
|
export interface FullTokenizer extends BasicTokenizer {
|
8
22
|
/**
|
9
|
-
*
|
23
|
+
* Asynchronously tokenize the given text, providing both the numeric tokens and their corresponding text.
|
24
|
+
*
|
25
|
+
* @param text - Input text string to be tokenized.
|
26
|
+
* @returns A promise containing an object with two arrays:
|
27
|
+
* 1. `tokens` - An array of numbers where each number is a token.
|
28
|
+
* 2. `tokenTexts` - An array of strings where each string represents the original text corresponding to each token.
|
10
29
|
*/
|
11
30
|
tokenizeWithTexts: (text: string) => PromiseLike<{
|
12
31
|
tokens: Array<number>;
|
13
32
|
tokenTexts: Array<string>;
|
14
33
|
}>;
|
15
34
|
/**
|
16
|
-
*
|
35
|
+
* Asynchronously revert a sequence of numeric tokens back into the original text.
|
36
|
+
* Detokenization is the process of transforming tokens back to a human-readable format, and it's essential in scenarios
|
37
|
+
* where the output needs to be interpretable or when the tokenization process has to be reversible.
|
38
|
+
*
|
39
|
+
* @param tokens - An array of numeric tokens to be converted back to text.
|
40
|
+
* @returns A promise containing a string that represents the original text corresponding to the sequence of input tokens.
|
17
41
|
*/
|
18
42
|
detokenize: (tokens: Array<number>) => PromiseLike<string>;
|
19
43
|
}
|
package/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
{
|
2
2
|
"name": "modelfusion",
|
3
3
|
"description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
|
4
|
-
"version": "0.
|
4
|
+
"version": "0.54.0",
|
5
5
|
"author": "Lars Grammel",
|
6
6
|
"license": "MIT",
|
7
7
|
"keywords": [
|
@@ -38,6 +38,16 @@
|
|
38
38
|
"types": "./index.d.ts",
|
39
39
|
"import": "./index.js",
|
40
40
|
"require": "./index.cjs"
|
41
|
+
},
|
42
|
+
"./browser": {
|
43
|
+
"types": "./browser/index.d.ts",
|
44
|
+
"import": "./browser/index.js",
|
45
|
+
"require": "./browser/index.cjs"
|
46
|
+
},
|
47
|
+
"./fastify-server": {
|
48
|
+
"types": "./server/fastify/index.d.ts",
|
49
|
+
"import": "./server/fastify/index.js",
|
50
|
+
"require": "./server/fastify/index.cjs"
|
41
51
|
}
|
42
52
|
},
|
43
53
|
"scripts": {
|
@@ -62,8 +72,6 @@
|
|
62
72
|
"zod-to-json-schema": "3.21.4"
|
63
73
|
},
|
64
74
|
"devDependencies": {
|
65
|
-
"@tsconfig/recommended": "1.0.3",
|
66
|
-
"@types/deep-equal": "^1.0.2",
|
67
75
|
"@types/node": "18.11.9",
|
68
76
|
"@types/ws": "^8.5.7",
|
69
77
|
"@typescript-eslint/eslint-plugin": "^6.1.0",
|
@@ -71,8 +79,9 @@
|
|
71
79
|
"copyfiles": "2.4.1",
|
72
80
|
"eslint": "^8.45.0",
|
73
81
|
"eslint-config-prettier": "9.0.0",
|
82
|
+
"fastify": "^4.0.0",
|
74
83
|
"husky": "^8.0.3",
|
75
|
-
"lint-staged": "
|
84
|
+
"lint-staged": "15.0.2",
|
76
85
|
"prettier": "3.0.3",
|
77
86
|
"rimraf": "5.0.5",
|
78
87
|
"typescript": "5.2.2",
|
@@ -0,0 +1,17 @@
|
|
1
|
+
/// <reference types="node" />
|
2
|
+
import { FlowRun } from "./FlowRun.js";
|
3
|
+
export type Asset = {
|
4
|
+
data: Buffer;
|
5
|
+
contentType: string;
|
6
|
+
name: string;
|
7
|
+
};
|
8
|
+
export interface AssetStorage {
|
9
|
+
storeAsset(options: {
|
10
|
+
run: FlowRun<unknown>;
|
11
|
+
asset: Asset;
|
12
|
+
}): Promise<void>;
|
13
|
+
readAsset(options: {
|
14
|
+
run: FlowRun<unknown>;
|
15
|
+
assetName: string;
|
16
|
+
}): Promise<Asset | null>;
|
17
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
export {};
|
@@ -0,0 +1,22 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.DefaultFlow = void 0;
|
4
|
+
class DefaultFlow {
|
5
|
+
constructor({ schema, process, }) {
|
6
|
+
Object.defineProperty(this, "schema", {
|
7
|
+
enumerable: true,
|
8
|
+
configurable: true,
|
9
|
+
writable: true,
|
10
|
+
value: void 0
|
11
|
+
});
|
12
|
+
Object.defineProperty(this, "process", {
|
13
|
+
enumerable: true,
|
14
|
+
configurable: true,
|
15
|
+
writable: true,
|
16
|
+
value: void 0
|
17
|
+
});
|
18
|
+
this.schema = schema;
|
19
|
+
this.process = process;
|
20
|
+
}
|
21
|
+
}
|
22
|
+
exports.DefaultFlow = DefaultFlow;
|
@@ -0,0 +1,16 @@
|
|
1
|
+
import { FlowRun } from "./FlowRun.js";
|
2
|
+
import { FlowSchema } from "./FlowSchema.js";
|
3
|
+
export declare class DefaultFlow<INPUT, EVENT> {
|
4
|
+
readonly schema: FlowSchema<INPUT, EVENT>;
|
5
|
+
constructor({ schema, process, }: {
|
6
|
+
schema: FlowSchema<INPUT, EVENT>;
|
7
|
+
process: (options: {
|
8
|
+
input: INPUT;
|
9
|
+
run: FlowRun<EVENT>;
|
10
|
+
}) => Promise<void>;
|
11
|
+
});
|
12
|
+
process: (options: {
|
13
|
+
input: INPUT;
|
14
|
+
run: FlowRun<EVENT>;
|
15
|
+
}) => Promise<void>;
|
16
|
+
}
|
@@ -0,0 +1,18 @@
|
|
1
|
+
export class DefaultFlow {
|
2
|
+
constructor({ schema, process, }) {
|
3
|
+
Object.defineProperty(this, "schema", {
|
4
|
+
enumerable: true,
|
5
|
+
configurable: true,
|
6
|
+
writable: true,
|
7
|
+
value: void 0
|
8
|
+
});
|
9
|
+
Object.defineProperty(this, "process", {
|
10
|
+
enumerable: true,
|
11
|
+
configurable: true,
|
12
|
+
writable: true,
|
13
|
+
value: void 0
|
14
|
+
});
|
15
|
+
this.schema = schema;
|
16
|
+
this.process = process;
|
17
|
+
}
|
18
|
+
}
|
@@ -0,0 +1,60 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.FileSystemAssetStorage = void 0;
|
4
|
+
const node_fs_1 = require("node:fs");
|
5
|
+
const node_path_1 = require("node:path");
|
6
|
+
class FileSystemAssetStorage {
|
7
|
+
constructor({ path, logger, }) {
|
8
|
+
Object.defineProperty(this, "path", {
|
9
|
+
enumerable: true,
|
10
|
+
configurable: true,
|
11
|
+
writable: true,
|
12
|
+
value: void 0
|
13
|
+
});
|
14
|
+
Object.defineProperty(this, "logger", {
|
15
|
+
enumerable: true,
|
16
|
+
configurable: true,
|
17
|
+
writable: true,
|
18
|
+
value: void 0
|
19
|
+
});
|
20
|
+
this.path = path;
|
21
|
+
this.logger = logger;
|
22
|
+
}
|
23
|
+
async storeAsset({ run, asset, }) {
|
24
|
+
try {
|
25
|
+
const assetPath = this.path(run);
|
26
|
+
await node_fs_1.promises.mkdir(assetPath, { recursive: true });
|
27
|
+
await node_fs_1.promises.writeFile((0, node_path_1.join)(assetPath, asset.name), asset.data);
|
28
|
+
await node_fs_1.promises.writeFile((0, node_path_1.join)(assetPath, `${asset.name}.meta.json`), JSON.stringify({
|
29
|
+
name: asset.name,
|
30
|
+
contentType: asset.contentType,
|
31
|
+
}));
|
32
|
+
}
|
33
|
+
catch (error) {
|
34
|
+
this.logger.logError({
|
35
|
+
run,
|
36
|
+
message: `Failed to store asset ${asset.name}`,
|
37
|
+
error,
|
38
|
+
});
|
39
|
+
throw error;
|
40
|
+
}
|
41
|
+
}
|
42
|
+
async readAsset(options) {
|
43
|
+
try {
|
44
|
+
const assetPath = this.path(options.run);
|
45
|
+
const data = await node_fs_1.promises.readFile((0, node_path_1.join)(assetPath, options.assetName));
|
46
|
+
const meta = await node_fs_1.promises.readFile((0, node_path_1.join)(assetPath, `${options.assetName}.meta.json`));
|
47
|
+
const { name, contentType } = JSON.parse(meta.toString());
|
48
|
+
return { data, name, contentType };
|
49
|
+
}
|
50
|
+
catch (error) {
|
51
|
+
this.logger.logError({
|
52
|
+
run: options.run,
|
53
|
+
message: `Failed to read asset ${options.assetName}`,
|
54
|
+
error,
|
55
|
+
});
|
56
|
+
throw error;
|
57
|
+
}
|
58
|
+
}
|
59
|
+
}
|
60
|
+
exports.FileSystemAssetStorage = FileSystemAssetStorage;
|
@@ -0,0 +1,19 @@
|
|
1
|
+
import type { Asset, AssetStorage } from "./AssetStorage.js";
|
2
|
+
import { FlowRun } from "./FlowRun.js";
|
3
|
+
import { Logger } from "./Logger.js";
|
4
|
+
export declare class FileSystemAssetStorage implements AssetStorage {
|
5
|
+
private readonly path;
|
6
|
+
private readonly logger;
|
7
|
+
constructor({ path, logger, }: {
|
8
|
+
path: (run: FlowRun<unknown>) => string;
|
9
|
+
logger: Logger;
|
10
|
+
});
|
11
|
+
storeAsset({ run, asset, }: {
|
12
|
+
run: FlowRun<unknown>;
|
13
|
+
asset: Asset;
|
14
|
+
}): Promise<void>;
|
15
|
+
readAsset(options: {
|
16
|
+
run: FlowRun<unknown>;
|
17
|
+
assetName: string;
|
18
|
+
}): Promise<Asset | null>;
|
19
|
+
}
|