modelfusion 0.48.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +186 -183
- package/core/FunctionEvent.d.ts +9 -1
- package/core/FunctionOptions.d.ts +4 -0
- package/core/executeFunctionCall.cjs +85 -0
- package/core/executeFunctionCall.d.ts +10 -0
- package/core/executeFunctionCall.js +81 -0
- package/guard/GuardEvent.d.ts +7 -0
- package/guard/guard.cjs +60 -54
- package/guard/guard.d.ts +2 -1
- package/guard/guard.js +60 -54
- package/guard/index.cjs +1 -0
- package/guard/index.d.ts +1 -0
- package/guard/index.js +1 -0
- package/model-function/AsyncIterableResultPromise.cjs +5 -5
- package/model-function/AsyncIterableResultPromise.d.ts +3 -3
- package/model-function/AsyncIterableResultPromise.js +5 -5
- package/model-function/Model.d.ts +1 -1
- package/model-function/ModelCallEvent.d.ts +5 -7
- package/model-function/embed/EmbeddingEvent.d.ts +2 -2
- package/model-function/embed/embed.cjs +5 -5
- package/model-function/embed/embed.js +5 -5
- package/model-function/{executeCall.cjs → executeStandardCall.cjs} +5 -3
- package/model-function/{executeCall.d.ts → executeStandardCall.d.ts} +1 -1
- package/model-function/{executeCall.js → executeStandardCall.js} +3 -1
- package/model-function/executeStreamCall.cjs +134 -0
- package/model-function/executeStreamCall.d.ts +20 -0
- package/model-function/executeStreamCall.js +130 -0
- package/model-function/generate-image/ImageGenerationEvent.d.ts +2 -2
- package/model-function/generate-image/generateImage.cjs +3 -3
- package/model-function/generate-image/generateImage.js +3 -3
- package/model-function/generate-speech/SpeechGenerationEvent.d.ts +27 -0
- package/model-function/generate-speech/SpeechGenerationModel.d.ts +15 -0
- package/model-function/generate-speech/generateSpeech.cjs +24 -0
- package/model-function/generate-speech/generateSpeech.d.ts +8 -0
- package/model-function/generate-speech/generateSpeech.js +20 -0
- package/model-function/generate-speech/index.cjs +20 -0
- package/model-function/generate-speech/index.d.ts +4 -0
- package/model-function/generate-speech/index.js +4 -0
- package/model-function/generate-speech/streamSpeech.cjs +34 -0
- package/model-function/generate-speech/streamSpeech.d.ts +8 -0
- package/model-function/generate-speech/streamSpeech.js +30 -0
- package/model-function/generate-structure/StructureGenerationEvent.d.ts +2 -2
- package/model-function/generate-structure/StructureStreamingEvent.d.ts +2 -2
- package/model-function/generate-structure/generateStructure.cjs +3 -3
- package/model-function/generate-structure/generateStructure.js +3 -3
- package/model-function/generate-structure/generateStructureOrText.cjs +3 -3
- package/model-function/generate-structure/generateStructureOrText.js +3 -3
- package/model-function/generate-structure/index.cjs +27 -0
- package/model-function/generate-structure/index.d.ts +11 -0
- package/model-function/generate-structure/index.js +11 -0
- package/model-function/generate-structure/streamStructure.cjs +29 -137
- package/model-function/generate-structure/streamStructure.js +28 -136
- package/model-function/generate-text/TextGenerationEvent.d.ts +8 -2
- package/model-function/generate-text/generateText.cjs +4 -4
- package/model-function/generate-text/generateText.d.ts +1 -1
- package/model-function/generate-text/generateText.js +4 -4
- package/model-function/generate-text/index.cjs +0 -1
- package/model-function/generate-text/index.d.ts +0 -1
- package/model-function/generate-text/index.js +0 -1
- package/model-function/generate-text/streamText.cjs +22 -129
- package/model-function/generate-text/streamText.js +21 -128
- package/model-function/generate-text/trimChatPrompt.cjs +1 -1
- package/model-function/generate-text/trimChatPrompt.d.ts +1 -1
- package/model-function/generate-text/trimChatPrompt.js +1 -1
- package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.d.ts +2 -2
- package/model-function/{transcribe-speech/transcribe.cjs → generate-transcription/generateTranscription.cjs} +7 -7
- package/model-function/{transcribe-speech/transcribe.d.ts → generate-transcription/generateTranscription.d.ts} +2 -2
- package/model-function/{transcribe-speech/transcribe.js → generate-transcription/generateTranscription.js} +5 -5
- package/model-function/index.cjs +5 -20
- package/model-function/index.d.ts +5 -20
- package/model-function/index.js +5 -20
- package/model-provider/elevenlabs/{ElevenLabsSpeechSynthesisModel.cjs → ElevenLabsSpeechModel.cjs} +6 -6
- package/model-provider/elevenlabs/{ElevenLabsSpeechSynthesisModel.d.ts → ElevenLabsSpeechModel.d.ts} +8 -8
- package/model-provider/elevenlabs/{ElevenLabsSpeechSynthesisModel.js → ElevenLabsSpeechModel.js} +4 -4
- package/model-provider/elevenlabs/index.cjs +1 -1
- package/model-provider/elevenlabs/index.d.ts +1 -1
- package/model-provider/elevenlabs/index.js +1 -1
- package/model-provider/huggingface/HuggingFaceImageDescriptionModel.cjs +21 -2
- package/model-provider/huggingface/HuggingFaceImageDescriptionModel.d.ts +11 -6
- package/model-provider/huggingface/HuggingFaceImageDescriptionModel.js +21 -2
- package/model-provider/lmnt/{LmntSpeechSynthesisModel.cjs → LmntSpeechModel.cjs} +5 -5
- package/model-provider/lmnt/LmntSpeechModel.d.ts +26 -0
- package/model-provider/lmnt/{LmntSpeechSynthesisModel.js → LmntSpeechModel.js} +3 -3
- package/model-provider/lmnt/index.cjs +1 -1
- package/model-provider/lmnt/index.d.ts +1 -1
- package/model-provider/lmnt/index.js +1 -1
- package/model-provider/openai/{OpenAITextGenerationModel.cjs → OpenAICompletionModel.cjs} +17 -17
- package/model-provider/openai/{OpenAITextGenerationModel.d.ts → OpenAICompletionModel.d.ts} +29 -29
- package/model-provider/openai/{OpenAITextGenerationModel.js → OpenAICompletionModel.js} +12 -12
- package/model-provider/openai/OpenAICostCalculator.cjs +8 -8
- package/model-provider/openai/OpenAICostCalculator.js +8 -8
- package/model-provider/openai/OpenAITextEmbeddingModel.d.ts +3 -3
- package/model-provider/openai/OpenAITranscriptionModel.d.ts +1 -1
- package/model-provider/openai/TikTokenTokenizer.d.ts +2 -2
- package/model-provider/openai/chat/OpenAIChatModel.d.ts +7 -7
- package/model-provider/openai/index.cjs +1 -1
- package/model-provider/openai/index.d.ts +1 -1
- package/model-provider/openai/index.js +1 -1
- package/package.json +1 -1
- package/retriever/retrieve.cjs +7 -75
- package/retriever/retrieve.js +7 -75
- package/tool/UseToolEvent.d.ts +7 -0
- package/tool/UseToolOrGenerateTextEvent.d.ts +7 -0
- package/tool/executeTool.cjs +2 -0
- package/tool/executeTool.js +2 -0
- package/tool/index.cjs +2 -0
- package/tool/index.d.ts +2 -0
- package/tool/index.js +2 -0
- package/tool/useTool.cjs +18 -10
- package/tool/useTool.js +18 -10
- package/tool/useToolOrGenerateText.cjs +34 -26
- package/tool/useToolOrGenerateText.js +34 -26
- package/vector-index/UpsertIntoVectorIndexEvent.cjs +2 -0
- package/vector-index/UpsertIntoVectorIndexEvent.d.ts +9 -0
- package/vector-index/UpsertIntoVectorIndexEvent.js +1 -0
- package/vector-index/VectorIndexRetriever.cjs +1 -4
- package/vector-index/VectorIndexRetriever.js +1 -4
- package/vector-index/index.cjs +1 -0
- package/vector-index/index.d.ts +1 -0
- package/vector-index/index.js +1 -0
- package/vector-index/upsertIntoVectorIndex.cjs +16 -7
- package/vector-index/upsertIntoVectorIndex.js +16 -7
- package/model-function/describe-image/ImageDescriptionEvent.d.ts +0 -18
- package/model-function/describe-image/ImageDescriptionModel.d.ts +0 -10
- package/model-function/describe-image/describeImage.cjs +0 -26
- package/model-function/describe-image/describeImage.d.ts +0 -9
- package/model-function/describe-image/describeImage.js +0 -22
- package/model-function/generate-text/TextStreamingEvent.d.ts +0 -7
- package/model-function/synthesize-speech/SpeechSynthesisEvent.d.ts +0 -21
- package/model-function/synthesize-speech/SpeechSynthesisModel.d.ts +0 -15
- package/model-function/synthesize-speech/synthesizeSpeech.cjs +0 -67
- package/model-function/synthesize-speech/synthesizeSpeech.d.ts +0 -14
- package/model-function/synthesize-speech/synthesizeSpeech.js +0 -63
- package/model-provider/lmnt/LmntSpeechSynthesisModel.d.ts +0 -26
- /package/{model-function/describe-image/ImageDescriptionEvent.cjs → guard/GuardEvent.cjs} +0 -0
- /package/{model-function/describe-image/ImageDescriptionEvent.js → guard/GuardEvent.js} +0 -0
- /package/model-function/{describe-image/ImageDescriptionModel.cjs → generate-speech/SpeechGenerationEvent.cjs} +0 -0
- /package/model-function/{describe-image/ImageDescriptionModel.js → generate-speech/SpeechGenerationEvent.js} +0 -0
- /package/model-function/{generate-text/TextStreamingEvent.cjs → generate-speech/SpeechGenerationModel.cjs} +0 -0
- /package/model-function/{generate-text/TextStreamingEvent.js → generate-speech/SpeechGenerationModel.js} +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.cjs +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionEvent.js +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.cjs +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.d.ts +0 -0
- /package/model-function/{transcribe-speech → generate-transcription}/TranscriptionModel.js +0 -0
- /package/{model-function/synthesize-speech/SpeechSynthesisEvent.cjs → tool/UseToolEvent.cjs} +0 -0
- /package/{model-function/synthesize-speech/SpeechSynthesisEvent.js → tool/UseToolEvent.js} +0 -0
- /package/{model-function/synthesize-speech/SpeechSynthesisModel.cjs → tool/UseToolOrGenerateTextEvent.cjs} +0 -0
- /package/{model-function/synthesize-speech/SpeechSynthesisModel.js → tool/UseToolOrGenerateTextEvent.js} +0 -0
package/tool/useTool.cjs
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
3
|
exports.useTool = void 0;
|
4
|
+
const executeFunctionCall_js_1 = require("../core/executeFunctionCall.cjs");
|
4
5
|
const generateStructure_js_1 = require("../model-function/generate-structure/generateStructure.cjs");
|
5
6
|
const executeTool_js_1 = require("./executeTool.cjs");
|
6
7
|
/**
|
@@ -15,15 +16,22 @@ async function useTool(model, tool, prompt, options) {
|
|
15
16
|
const expandedPrompt = typeof prompt === "function"
|
16
17
|
? prompt(tool)
|
17
18
|
: prompt;
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
19
|
+
return (0, executeFunctionCall_js_1.executeFunctionCall)({
|
20
|
+
options,
|
21
|
+
input: expandedPrompt,
|
22
|
+
functionType: "use-tool",
|
23
|
+
execute: async (options) => {
|
24
|
+
const { value } = await (0, generateStructure_js_1.generateStructure)(model, {
|
25
|
+
name: tool.name,
|
26
|
+
description: tool.description,
|
27
|
+
schema: tool.inputSchema,
|
28
|
+
}, expandedPrompt, options).asFullResponse();
|
29
|
+
return {
|
30
|
+
tool: tool.name,
|
31
|
+
parameters: value,
|
32
|
+
result: await (0, executeTool_js_1.executeTool)(tool, value, options),
|
33
|
+
};
|
34
|
+
},
|
35
|
+
});
|
28
36
|
}
|
29
37
|
exports.useTool = useTool;
|
package/tool/useTool.js
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
import { executeFunctionCall } from "../core/executeFunctionCall.js";
|
1
2
|
import { generateStructure } from "../model-function/generate-structure/generateStructure.js";
|
2
3
|
import { executeTool } from "./executeTool.js";
|
3
4
|
/**
|
@@ -12,14 +13,21 @@ export async function useTool(model, tool, prompt, options) {
|
|
12
13
|
const expandedPrompt = typeof prompt === "function"
|
13
14
|
? prompt(tool)
|
14
15
|
: prompt;
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
16
|
+
return executeFunctionCall({
|
17
|
+
options,
|
18
|
+
input: expandedPrompt,
|
19
|
+
functionType: "use-tool",
|
20
|
+
execute: async (options) => {
|
21
|
+
const { value } = await generateStructure(model, {
|
22
|
+
name: tool.name,
|
23
|
+
description: tool.description,
|
24
|
+
schema: tool.inputSchema,
|
25
|
+
}, expandedPrompt, options).asFullResponse();
|
26
|
+
return {
|
27
|
+
tool: tool.name,
|
28
|
+
parameters: value,
|
29
|
+
result: await executeTool(tool, value, options),
|
30
|
+
};
|
31
|
+
},
|
32
|
+
});
|
25
33
|
}
|
@@ -1,6 +1,7 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
3
|
exports.useToolOrGenerateText = void 0;
|
4
|
+
const executeFunctionCall_js_1 = require("../core/executeFunctionCall.cjs");
|
4
5
|
const generateStructureOrText_js_1 = require("../model-function/generate-structure/generateStructureOrText.cjs");
|
5
6
|
const NoSuchToolError_js_1 = require("./NoSuchToolError.cjs");
|
6
7
|
const executeTool_js_1 = require("./executeTool.cjs");
|
@@ -9,31 +10,38 @@ async function useToolOrGenerateText(model, tools, prompt, options) {
|
|
9
10
|
const expandedPrompt = typeof prompt === "function"
|
10
11
|
? prompt(tools)
|
11
12
|
: prompt;
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
13
|
+
return (0, executeFunctionCall_js_1.executeFunctionCall)({
|
14
|
+
options,
|
15
|
+
input: expandedPrompt,
|
16
|
+
functionType: "use-tool-or-generate-text",
|
17
|
+
execute: async (options) => {
|
18
|
+
const modelResponse = await (0, generateStructureOrText_js_1.generateStructureOrText)(model, tools.map((tool) => ({
|
19
|
+
name: tool.name,
|
20
|
+
description: tool.description,
|
21
|
+
schema: tool.inputSchema,
|
22
|
+
})), expandedPrompt, options);
|
23
|
+
const { structure, text } = modelResponse;
|
24
|
+
if (structure == null) {
|
25
|
+
return {
|
26
|
+
tool: null,
|
27
|
+
parameters: null,
|
28
|
+
result: null,
|
29
|
+
text,
|
30
|
+
};
|
31
|
+
}
|
32
|
+
const tool = tools.find((tool) => tool.name === structure);
|
33
|
+
if (tool == null) {
|
34
|
+
throw new NoSuchToolError_js_1.NoSuchToolError(structure.toString());
|
35
|
+
}
|
36
|
+
const toolParameters = modelResponse.value;
|
37
|
+
const result = await (0, executeTool_js_1.executeTool)(tool, toolParameters, options);
|
38
|
+
return {
|
39
|
+
tool: structure,
|
40
|
+
result,
|
41
|
+
parameters: toolParameters,
|
42
|
+
text: text, // string | null is the expected value here
|
43
|
+
};
|
44
|
+
},
|
45
|
+
});
|
38
46
|
}
|
39
47
|
exports.useToolOrGenerateText = useToolOrGenerateText;
|
@@ -1,3 +1,4 @@
|
|
1
|
+
import { executeFunctionCall } from "../core/executeFunctionCall.js";
|
1
2
|
import { generateStructureOrText } from "../model-function/generate-structure/generateStructureOrText.js";
|
2
3
|
import { NoSuchToolError } from "./NoSuchToolError.js";
|
3
4
|
import { executeTool } from "./executeTool.js";
|
@@ -6,30 +7,37 @@ export async function useToolOrGenerateText(model, tools, prompt, options) {
|
|
6
7
|
const expandedPrompt = typeof prompt === "function"
|
7
8
|
? prompt(tools)
|
8
9
|
: prompt;
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
10
|
+
return executeFunctionCall({
|
11
|
+
options,
|
12
|
+
input: expandedPrompt,
|
13
|
+
functionType: "use-tool-or-generate-text",
|
14
|
+
execute: async (options) => {
|
15
|
+
const modelResponse = await generateStructureOrText(model, tools.map((tool) => ({
|
16
|
+
name: tool.name,
|
17
|
+
description: tool.description,
|
18
|
+
schema: tool.inputSchema,
|
19
|
+
})), expandedPrompt, options);
|
20
|
+
const { structure, text } = modelResponse;
|
21
|
+
if (structure == null) {
|
22
|
+
return {
|
23
|
+
tool: null,
|
24
|
+
parameters: null,
|
25
|
+
result: null,
|
26
|
+
text,
|
27
|
+
};
|
28
|
+
}
|
29
|
+
const tool = tools.find((tool) => tool.name === structure);
|
30
|
+
if (tool == null) {
|
31
|
+
throw new NoSuchToolError(structure.toString());
|
32
|
+
}
|
33
|
+
const toolParameters = modelResponse.value;
|
34
|
+
const result = await executeTool(tool, toolParameters, options);
|
35
|
+
return {
|
36
|
+
tool: structure,
|
37
|
+
result,
|
38
|
+
parameters: toolParameters,
|
39
|
+
text: text, // string | null is the expected value here
|
40
|
+
};
|
41
|
+
},
|
42
|
+
});
|
35
43
|
}
|
@@ -0,0 +1,9 @@
|
|
1
|
+
import { BaseFunctionFinishedEvent, BaseFunctionStartedEvent } from "../core/FunctionEvent.js";
|
2
|
+
export interface UpsertIntoVectorIndexStartedEvent extends BaseFunctionStartedEvent {
|
3
|
+
functionType: "upsert-into-vector-index";
|
4
|
+
objects: unknown[];
|
5
|
+
}
|
6
|
+
export interface UpsertIntoVectorIndexFinishedEvent extends BaseFunctionFinishedEvent {
|
7
|
+
functionType: "upsert-into-vector-index";
|
8
|
+
objects: unknown[];
|
9
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
export {};
|
@@ -31,10 +31,7 @@ class VectorIndexRetriever {
|
|
31
31
|
};
|
32
32
|
}
|
33
33
|
async retrieve(query, options) {
|
34
|
-
const embedding = await (0, embed_js_1.embed)(this.embeddingModel, query,
|
35
|
-
functionId: options?.functionId,
|
36
|
-
run: options?.run,
|
37
|
-
});
|
34
|
+
const embedding = await (0, embed_js_1.embed)(this.embeddingModel, query, options);
|
38
35
|
const queryResult = await this.vectorIndex.queryByVector({
|
39
36
|
queryVector: embedding,
|
40
37
|
maxResults: this.settings.maxResults ?? 1,
|
@@ -28,10 +28,7 @@ export class VectorIndexRetriever {
|
|
28
28
|
};
|
29
29
|
}
|
30
30
|
async retrieve(query, options) {
|
31
|
-
const embedding = await embed(this.embeddingModel, query,
|
32
|
-
functionId: options?.functionId,
|
33
|
-
run: options?.run,
|
34
|
-
});
|
31
|
+
const embedding = await embed(this.embeddingModel, query, options);
|
35
32
|
const queryResult = await this.vectorIndex.queryByVector({
|
36
33
|
queryVector: embedding,
|
37
34
|
maxResults: this.settings.maxResults ?? 1,
|
package/vector-index/index.cjs
CHANGED
@@ -14,6 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
15
15
|
};
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
17
|
+
__exportStar(require("./UpsertIntoVectorIndexEvent.cjs"), exports);
|
17
18
|
__exportStar(require("./VectorIndex.cjs"), exports);
|
18
19
|
__exportStar(require("./VectorIndexRetriever.cjs"), exports);
|
19
20
|
__exportStar(require("./memory/MemoryVectorIndex.cjs"), exports);
|
package/vector-index/index.d.ts
CHANGED
package/vector-index/index.js
CHANGED
@@ -2,14 +2,23 @@
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
3
|
exports.upsertIntoVectorIndex = void 0;
|
4
4
|
const nanoid_1 = require("nanoid");
|
5
|
+
const executeFunctionCall_js_1 = require("../core/executeFunctionCall.cjs");
|
5
6
|
const embed_js_1 = require("../model-function/embed/embed.cjs");
|
6
7
|
async function upsertIntoVectorIndex({ vectorIndex, embeddingModel, generateId = nanoid_1.nanoid, objects, getValueToEmbed, getId, }, options) {
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
8
|
+
return (0, executeFunctionCall_js_1.executeFunctionCall)({
|
9
|
+
options,
|
10
|
+
input: objects,
|
11
|
+
functionType: "upsert-into-vector-index",
|
12
|
+
inputPropertyName: "objects",
|
13
|
+
execute: async (options) => {
|
14
|
+
// many embedding models support bulk embedding, so we first embed all texts:
|
15
|
+
const embeddings = await (0, embed_js_1.embedMany)(embeddingModel, objects.map(getValueToEmbed), options);
|
16
|
+
await vectorIndex.upsertMany(objects.map((object, i) => ({
|
17
|
+
id: getId?.(object, i) ?? generateId(),
|
18
|
+
vector: embeddings[i],
|
19
|
+
data: object,
|
20
|
+
})));
|
21
|
+
},
|
22
|
+
});
|
14
23
|
}
|
15
24
|
exports.upsertIntoVectorIndex = upsertIntoVectorIndex;
|
@@ -1,11 +1,20 @@
|
|
1
1
|
import { nanoid as createId } from "nanoid";
|
2
|
+
import { executeFunctionCall } from "../core/executeFunctionCall.js";
|
2
3
|
import { embedMany } from "../model-function/embed/embed.js";
|
3
4
|
export async function upsertIntoVectorIndex({ vectorIndex, embeddingModel, generateId = createId, objects, getValueToEmbed, getId, }, options) {
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
5
|
+
return executeFunctionCall({
|
6
|
+
options,
|
7
|
+
input: objects,
|
8
|
+
functionType: "upsert-into-vector-index",
|
9
|
+
inputPropertyName: "objects",
|
10
|
+
execute: async (options) => {
|
11
|
+
// many embedding models support bulk embedding, so we first embed all texts:
|
12
|
+
const embeddings = await embedMany(embeddingModel, objects.map(getValueToEmbed), options);
|
13
|
+
await vectorIndex.upsertMany(objects.map((object, i) => ({
|
14
|
+
id: getId?.(object, i) ?? generateId(),
|
15
|
+
vector: embeddings[i],
|
16
|
+
data: object,
|
17
|
+
})));
|
18
|
+
},
|
19
|
+
});
|
11
20
|
}
|
@@ -1,18 +0,0 @@
|
|
1
|
-
import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelCallEvent.js";
|
2
|
-
export interface ImageDescriptionStartedEvent extends BaseModelCallStartedEvent {
|
3
|
-
functionType: "image-description";
|
4
|
-
}
|
5
|
-
export type ImageDescriptionFinishedEventResult = {
|
6
|
-
status: "success";
|
7
|
-
response: unknown;
|
8
|
-
value: string;
|
9
|
-
} | {
|
10
|
-
status: "error";
|
11
|
-
error: unknown;
|
12
|
-
} | {
|
13
|
-
status: "abort";
|
14
|
-
};
|
15
|
-
export interface ImageDescriptionFinishedEvent extends BaseModelCallFinishedEvent {
|
16
|
-
functionType: "image-description";
|
17
|
-
result: ImageDescriptionFinishedEventResult;
|
18
|
-
}
|
@@ -1,10 +0,0 @@
|
|
1
|
-
import { FunctionOptions } from "../../core/FunctionOptions.js";
|
2
|
-
import { Model, ModelSettings } from "../Model.js";
|
3
|
-
export interface ImageDescriptionModelSettings extends ModelSettings {
|
4
|
-
}
|
5
|
-
export interface ImageDescriptionModel<DATA, SETTINGS extends ImageDescriptionModelSettings = ImageDescriptionModelSettings> extends Model<SETTINGS> {
|
6
|
-
doDescribeImage: (data: DATA, options?: FunctionOptions) => PromiseLike<{
|
7
|
-
response: unknown;
|
8
|
-
description: string;
|
9
|
-
}>;
|
10
|
-
}
|
@@ -1,26 +0,0 @@
|
|
1
|
-
"use strict";
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.describeImage = void 0;
|
4
|
-
const executeCall_js_1 = require("../executeCall.cjs");
|
5
|
-
const ModelFunctionPromise_js_1 = require("../ModelFunctionPromise.cjs");
|
6
|
-
/**
|
7
|
-
* Describe an image as text.
|
8
|
-
*
|
9
|
-
* Depending on the model, this can be used for image captioning, for describing the contents of an image, or for OCR.
|
10
|
-
*/
|
11
|
-
function describeImage(model, data, options) {
|
12
|
-
return new ModelFunctionPromise_js_1.ModelFunctionPromise((0, executeCall_js_1.executeCall)({
|
13
|
-
functionType: "image-description",
|
14
|
-
input: data,
|
15
|
-
model,
|
16
|
-
options,
|
17
|
-
generateResponse: async (options) => {
|
18
|
-
const result = await model.doDescribeImage(data, options);
|
19
|
-
return {
|
20
|
-
response: result.response,
|
21
|
-
extractedValue: result.description,
|
22
|
-
};
|
23
|
-
},
|
24
|
-
}));
|
25
|
-
}
|
26
|
-
exports.describeImage = describeImage;
|
@@ -1,9 +0,0 @@
|
|
1
|
-
import { FunctionOptions } from "../../core/FunctionOptions.js";
|
2
|
-
import { ModelFunctionPromise } from "../ModelFunctionPromise.js";
|
3
|
-
import { ImageDescriptionModel, ImageDescriptionModelSettings } from "./ImageDescriptionModel.js";
|
4
|
-
/**
|
5
|
-
* Describe an image as text.
|
6
|
-
*
|
7
|
-
* Depending on the model, this can be used for image captioning, for describing the contents of an image, or for OCR.
|
8
|
-
*/
|
9
|
-
export declare function describeImage<DATA>(model: ImageDescriptionModel<DATA, ImageDescriptionModelSettings>, data: DATA, options?: FunctionOptions): ModelFunctionPromise<string>;
|
@@ -1,22 +0,0 @@
|
|
1
|
-
import { executeCall } from "../executeCall.js";
|
2
|
-
import { ModelFunctionPromise } from "../ModelFunctionPromise.js";
|
3
|
-
/**
|
4
|
-
* Describe an image as text.
|
5
|
-
*
|
6
|
-
* Depending on the model, this can be used for image captioning, for describing the contents of an image, or for OCR.
|
7
|
-
*/
|
8
|
-
export function describeImage(model, data, options) {
|
9
|
-
return new ModelFunctionPromise(executeCall({
|
10
|
-
functionType: "image-description",
|
11
|
-
input: data,
|
12
|
-
model,
|
13
|
-
options,
|
14
|
-
generateResponse: async (options) => {
|
15
|
-
const result = await model.doDescribeImage(data, options);
|
16
|
-
return {
|
17
|
-
response: result.response,
|
18
|
-
extractedValue: result.description,
|
19
|
-
};
|
20
|
-
},
|
21
|
-
}));
|
22
|
-
}
|
@@ -1,7 +0,0 @@
|
|
1
|
-
import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelCallEvent.js";
|
2
|
-
export interface TextStreamingStartedEvent extends BaseModelCallStartedEvent {
|
3
|
-
functionType: "text-streaming";
|
4
|
-
}
|
5
|
-
export interface TextStreamingFinishedEvent extends BaseModelCallFinishedEvent {
|
6
|
-
functionType: "text-streaming";
|
7
|
-
}
|
@@ -1,21 +0,0 @@
|
|
1
|
-
/// <reference types="node" />
|
2
|
-
import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelCallEvent.js";
|
3
|
-
export interface SpeechSynthesisStartedEvent extends BaseModelCallStartedEvent {
|
4
|
-
functionType: "speech-synthesis";
|
5
|
-
input: string;
|
6
|
-
}
|
7
|
-
export type SpeechSynthesisFinishedEventResult = {
|
8
|
-
status: "success";
|
9
|
-
response: unknown;
|
10
|
-
value: Buffer;
|
11
|
-
} | {
|
12
|
-
status: "error";
|
13
|
-
error: unknown;
|
14
|
-
} | {
|
15
|
-
status: "abort";
|
16
|
-
};
|
17
|
-
export interface SpeechSynthesisFinishedEvent extends BaseModelCallFinishedEvent {
|
18
|
-
functionType: "speech-synthesis";
|
19
|
-
input: string;
|
20
|
-
result: SpeechSynthesisFinishedEventResult;
|
21
|
-
}
|
@@ -1,15 +0,0 @@
|
|
1
|
-
/// <reference types="node" />
|
2
|
-
import { FunctionOptions } from "../../core/FunctionOptions.js";
|
3
|
-
import { Delta } from "../../model-function/Delta.js";
|
4
|
-
import { Model, ModelSettings } from "../Model.js";
|
5
|
-
export interface SpeechSynthesisModelSettings extends ModelSettings {
|
6
|
-
}
|
7
|
-
export interface SpeechSynthesisModel<SETTINGS extends SpeechSynthesisModelSettings = SpeechSynthesisModelSettings> extends Model<SETTINGS> {
|
8
|
-
/**
|
9
|
-
* Generates an mp3 audio buffer that contains the speech for the given text.
|
10
|
-
*/
|
11
|
-
doSynthesizeSpeechStandard(text: string, options?: FunctionOptions): PromiseLike<Buffer>;
|
12
|
-
}
|
13
|
-
export interface DuplexSpeechSynthesisModel<SETTINGS extends SpeechSynthesisModelSettings = SpeechSynthesisModelSettings> extends SpeechSynthesisModel<SETTINGS> {
|
14
|
-
doSynthesizeSpeechStreamDuplex(textStream: AsyncIterable<string>, options?: FunctionOptions): PromiseLike<AsyncIterable<Delta<Buffer>>>;
|
15
|
-
}
|
@@ -1,67 +0,0 @@
|
|
1
|
-
"use strict";
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.synthesizeSpeech = void 0;
|
4
|
-
const AsyncIterableResultPromise_js_1 = require("../../model-function/AsyncIterableResultPromise.cjs");
|
5
|
-
const ModelFunctionPromise_js_1 = require("../ModelFunctionPromise.cjs");
|
6
|
-
const executeCall_js_1 = require("../executeCall.cjs");
|
7
|
-
function synthesizeSpeech(model, text, options) {
|
8
|
-
const mode = options?.mode ?? "standard";
|
9
|
-
switch (mode) {
|
10
|
-
case "standard": {
|
11
|
-
if (typeof text !== "string") {
|
12
|
-
throw new Error(`The "standard" mode only supports a string input, but received ${text}`);
|
13
|
-
}
|
14
|
-
return new ModelFunctionPromise_js_1.ModelFunctionPromise((0, executeCall_js_1.executeCall)({
|
15
|
-
functionType: "speech-synthesis",
|
16
|
-
input: text,
|
17
|
-
model,
|
18
|
-
options,
|
19
|
-
generateResponse: async (options) => {
|
20
|
-
const response = await model.doSynthesizeSpeechStandard(text, options);
|
21
|
-
return {
|
22
|
-
response,
|
23
|
-
extractedValue: response,
|
24
|
-
};
|
25
|
-
},
|
26
|
-
}));
|
27
|
-
}
|
28
|
-
case "stream-duplex": {
|
29
|
-
if (typeof text === "string") {
|
30
|
-
throw new Error(`The "stream-duplex" mode only supports an AsyncIterable<string> input, but received ${text}`);
|
31
|
-
}
|
32
|
-
if (!("doSynthesizeSpeechStreamDuplex" in model) ||
|
33
|
-
typeof model.doSynthesizeSpeechStreamDuplex !== "function") {
|
34
|
-
throw new Error(`The "stream-duplex" mode is not supported by this model.`);
|
35
|
-
}
|
36
|
-
return new AsyncIterableResultPromise_js_1.AsyncIterableResultPromise(doSynthesizeSpeechStreamDuplex(model, text, options));
|
37
|
-
}
|
38
|
-
default: {
|
39
|
-
const mode_ = mode;
|
40
|
-
throw new Error(`Unsupported mode: ${mode_}`);
|
41
|
-
}
|
42
|
-
}
|
43
|
-
}
|
44
|
-
exports.synthesizeSpeech = synthesizeSpeech;
|
45
|
-
async function doSynthesizeSpeechStreamDuplex(model, text, options) {
|
46
|
-
const speechDeltas = await model.doSynthesizeSpeechStreamDuplex(text, options);
|
47
|
-
// Convert the speechDeltas (AsyncIterable<Delta<Buffer>>) to an AsyncIterable<Buffer>
|
48
|
-
const bufferStream = convertDeltasToBuffers(speechDeltas);
|
49
|
-
return {
|
50
|
-
output: bufferStream,
|
51
|
-
metadata: {
|
52
|
-
model: model.modelInformation,
|
53
|
-
callId: "test",
|
54
|
-
startTimestamp: new Date(),
|
55
|
-
},
|
56
|
-
};
|
57
|
-
}
|
58
|
-
async function* convertDeltasToBuffers(deltas) {
|
59
|
-
for await (const delta of deltas) {
|
60
|
-
switch (delta.type) {
|
61
|
-
case "error":
|
62
|
-
throw delta.error;
|
63
|
-
case "delta":
|
64
|
-
yield delta.valueDelta;
|
65
|
-
}
|
66
|
-
}
|
67
|
-
}
|
@@ -1,14 +0,0 @@
|
|
1
|
-
/// <reference types="node" />
|
2
|
-
import { FunctionOptions } from "../../core/FunctionOptions.js";
|
3
|
-
import { AsyncIterableResultPromise } from "../../model-function/AsyncIterableResultPromise.js";
|
4
|
-
import { ModelFunctionPromise } from "../ModelFunctionPromise.js";
|
5
|
-
import { DuplexSpeechSynthesisModel, SpeechSynthesisModel, SpeechSynthesisModelSettings } from "./SpeechSynthesisModel.js";
|
6
|
-
/**
|
7
|
-
* Synthesizes speech from text.
|
8
|
-
*/
|
9
|
-
export declare function synthesizeSpeech(model: SpeechSynthesisModel<SpeechSynthesisModelSettings>, text: string, options?: FunctionOptions & {
|
10
|
-
mode?: "standard";
|
11
|
-
}): ModelFunctionPromise<Buffer>;
|
12
|
-
export declare function synthesizeSpeech(model: DuplexSpeechSynthesisModel<SpeechSynthesisModelSettings>, text: AsyncIterable<string>, options: FunctionOptions & {
|
13
|
-
mode: "stream-duplex";
|
14
|
-
}): AsyncIterableResultPromise<Buffer>;
|
@@ -1,63 +0,0 @@
|
|
1
|
-
import { AsyncIterableResultPromise } from "../../model-function/AsyncIterableResultPromise.js";
|
2
|
-
import { ModelFunctionPromise } from "../ModelFunctionPromise.js";
|
3
|
-
import { executeCall } from "../executeCall.js";
|
4
|
-
export function synthesizeSpeech(model, text, options) {
|
5
|
-
const mode = options?.mode ?? "standard";
|
6
|
-
switch (mode) {
|
7
|
-
case "standard": {
|
8
|
-
if (typeof text !== "string") {
|
9
|
-
throw new Error(`The "standard" mode only supports a string input, but received ${text}`);
|
10
|
-
}
|
11
|
-
return new ModelFunctionPromise(executeCall({
|
12
|
-
functionType: "speech-synthesis",
|
13
|
-
input: text,
|
14
|
-
model,
|
15
|
-
options,
|
16
|
-
generateResponse: async (options) => {
|
17
|
-
const response = await model.doSynthesizeSpeechStandard(text, options);
|
18
|
-
return {
|
19
|
-
response,
|
20
|
-
extractedValue: response,
|
21
|
-
};
|
22
|
-
},
|
23
|
-
}));
|
24
|
-
}
|
25
|
-
case "stream-duplex": {
|
26
|
-
if (typeof text === "string") {
|
27
|
-
throw new Error(`The "stream-duplex" mode only supports an AsyncIterable<string> input, but received ${text}`);
|
28
|
-
}
|
29
|
-
if (!("doSynthesizeSpeechStreamDuplex" in model) ||
|
30
|
-
typeof model.doSynthesizeSpeechStreamDuplex !== "function") {
|
31
|
-
throw new Error(`The "stream-duplex" mode is not supported by this model.`);
|
32
|
-
}
|
33
|
-
return new AsyncIterableResultPromise(doSynthesizeSpeechStreamDuplex(model, text, options));
|
34
|
-
}
|
35
|
-
default: {
|
36
|
-
const mode_ = mode;
|
37
|
-
throw new Error(`Unsupported mode: ${mode_}`);
|
38
|
-
}
|
39
|
-
}
|
40
|
-
}
|
41
|
-
async function doSynthesizeSpeechStreamDuplex(model, text, options) {
|
42
|
-
const speechDeltas = await model.doSynthesizeSpeechStreamDuplex(text, options);
|
43
|
-
// Convert the speechDeltas (AsyncIterable<Delta<Buffer>>) to an AsyncIterable<Buffer>
|
44
|
-
const bufferStream = convertDeltasToBuffers(speechDeltas);
|
45
|
-
return {
|
46
|
-
output: bufferStream,
|
47
|
-
metadata: {
|
48
|
-
model: model.modelInformation,
|
49
|
-
callId: "test",
|
50
|
-
startTimestamp: new Date(),
|
51
|
-
},
|
52
|
-
};
|
53
|
-
}
|
54
|
-
async function* convertDeltasToBuffers(deltas) {
|
55
|
-
for await (const delta of deltas) {
|
56
|
-
switch (delta.type) {
|
57
|
-
case "error":
|
58
|
-
throw delta.error;
|
59
|
-
case "delta":
|
60
|
-
yield delta.valueDelta;
|
61
|
-
}
|
62
|
-
}
|
63
|
-
}
|