@llumiverse/drivers 0.8.3 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cjs/index.js +3 -2
- package/lib/cjs/index.js.map +1 -1
- package/lib/cjs/mistral/index.js +147 -0
- package/lib/cjs/mistral/index.js.map +1 -0
- package/lib/cjs/mistral/types.js +83 -0
- package/lib/cjs/mistral/types.js.map +1 -0
- package/lib/cjs/togetherai/index.js +5 -14
- package/lib/cjs/togetherai/index.js.map +1 -1
- package/lib/cjs/vertexai/models/palm-model-base.js +6 -14
- package/lib/cjs/vertexai/models/palm-model-base.js.map +1 -1
- package/lib/esm/index.js +3 -2
- package/lib/esm/index.js.map +1 -1
- package/lib/esm/mistral/index.js +143 -0
- package/lib/esm/mistral/index.js.map +1 -0
- package/lib/esm/mistral/types.js +80 -0
- package/lib/esm/mistral/types.js.map +1 -0
- package/lib/esm/togetherai/index.js +5 -14
- package/lib/esm/togetherai/index.js.map +1 -1
- package/lib/esm/vertexai/models/palm-model-base.js +5 -13
- package/lib/esm/vertexai/models/palm-model-base.js.map +1 -1
- package/lib/types/index.d.ts +3 -2
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/mistral/index.d.ts +32 -0
- package/lib/types/mistral/index.d.ts.map +1 -0
- package/lib/types/mistral/types.d.ts +131 -0
- package/lib/types/mistral/types.d.ts.map +1 -0
- package/lib/types/togetherai/index.d.ts.map +1 -1
- package/lib/types/vertexai/models/palm-model-base.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/index.ts +4 -2
- package/src/mistral/index.ts +193 -0
- package/src/mistral/types.ts +211 -0
- package/src/togetherai/index.ts +6 -14
- package/src/vertexai/models/palm-model-base.ts +6 -17
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/togetherai/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,cAAc,EAA+C,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACvH,OAAO,EAAE,WAAW,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/togetherai/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,cAAc,EAA+C,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACvH,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAO/C,MAAM,OAAO,gBAAiB,SAAQ,cAA+C;IACjF,QAAQ,CAAS;IACjB,MAAM,CAAS;IACf,aAAa,CAAgB;IAC7B,WAAW,CAAc;IAEzB,YAAY,OAAgC;QACxC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC,cAAc,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,0BAA0B,CAAC,CAAC,WAAW,CAAC;YACvE,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;SACzC,CAAC,CAAC;IACP,CAAC;IAED,iBAAiB,GAAG,CAAC,OAAyB,EAAE,EAAE;QAC9C,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;YACzB;gBACI,IAAI,EAAE,aAAa;gBACnB,MAAM,EAAE,OAAO,CAAC,YAAY;aAC/B,CAAC,CAAC,CAAC,SAAS,CAAC;IACtB,CAAC,CAAA;IAED,KAAK,CAAC,iBAAiB,CAAC,MAAc,EAAE,OAAyB;QAC7D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,iBAAiB,EAAE;YACvD,OAAO,EAAE;gBACL,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,MAAM;gBACd,eAAe,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC;gBAChD,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,IAAI;gBACtC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,GAAG;gBACvC,IAAI,EAAE;oBACF,MAAM;oBACN,SAAS;iBACZ;aACJ;SACJ,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;QAC9B,OAAO;YACH,MAAM,EAAE,IAAI;YACZ,WAAW,EAAE;gBACT,MAAM,EAAE,KAAK,CAAC,aAAa;gBAC3B,MAAM,EAAE,KAAK,CAAC,iBAAiB;gBAC/B,KAAK,EAAE,KAAK,CAAC,YAAY;aAC5B;SACJ,CAAA;IACL,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,MAAc,EAAE,OAAyB;QAEnE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC1D,OAAO,EAAE;gBACL,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,IAAI;gBACtC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,GAAG;gBACvC,eAAe,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC;gBAChD,MAAM,EAAE,IAAI;gBACZ,IAAI,EAAE;oBACF,MAAM;oBACN,SAAS;iBACZ;aACJ;YACD,MAAM,EAAE,KAAK;SAChB,CAAC,CAAA;QAEF,OAAO,kBAAkB,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;QACvC,CAAC,CAAC,CAAC;IAEP,CAAC;IAED,KAAK,CAAC,UAAU;QACZ,MAAM,MAAM,GAAwB,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC/E,6DAA6D;QAE7D,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YAC5B,OAAO;gBACH,EAAE,EAAE,CAAC,CAAC,IAAI;gBACV,IAAI,EAAE,CAAC,CAAC,YAAY;gBACpB,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,CAAC,aAAa,CAAC,cAAc,CAAC;aAC1C,CAAA;QACL,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IAEpB,CAAC;IAED,mBAAmB;QACf,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC/C,CAAC;IACD,kBAAkB;QACd,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC/C,CAAC;IACD,YAAY;IACZ,kBAAkB,CAAC,OAAe,EAAE,KAA0B;QAC1D,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC/C,CAAC;CAEJ"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { transformSSEStream } from "@llumiverse/core/async";
|
|
1
2
|
import { generateStreamingPrompt } from "../utils/tensor.js";
|
|
2
3
|
export class AbstractPalmModelDefinition {
|
|
3
4
|
createPrompt(driver, segments, options) {
|
|
@@ -42,19 +43,10 @@ export class AbstractPalmModelDefinition {
|
|
|
42
43
|
payload: newPrompt,
|
|
43
44
|
reader: 'sse'
|
|
44
45
|
});
|
|
45
|
-
return eventStrean
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
constructor(def) {
|
|
50
|
-
super({
|
|
51
|
-
transform(event, controller) {
|
|
52
|
-
if (event.type === 'event' && event.data) {
|
|
53
|
-
const data = JSON.parse(event.data);
|
|
54
|
-
const stringChunk = def.extractContentFromResponseChunk(data);
|
|
55
|
-
controller.enqueue(Array.isArray(stringChunk) ? stringChunk.join('') : stringChunk);
|
|
56
|
-
}
|
|
57
|
-
}
|
|
46
|
+
return transformSSEStream(eventStrean, (data) => {
|
|
47
|
+
const json = JSON.parse(data);
|
|
48
|
+
const stringChunk = this.extractContentFromResponseChunk(json);
|
|
49
|
+
return Array.isArray(stringChunk) ? stringChunk.join('') : stringChunk;
|
|
58
50
|
});
|
|
59
51
|
}
|
|
60
52
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"palm-model-base.js","sourceRoot":"","sources":["../../../../src/vertexai/models/palm-model-base.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"palm-model-base.js","sourceRoot":"","sources":["../../../../src/vertexai/models/palm-model-base.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAI5D,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAiC7D,MAAM,OAAgB,2BAA2B;IAY7C,YAAY,CAAC,MAAsB,EAAE,QAAyB,EAAE,OAAsB;QAClF,OAAO,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,MAAsB,EAAE,MAA8C,EAAE,OAAyB;QACrH,MAAM,kBAAkB,GAAG,MAA6B,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,UAAU,EAAE;YAC3C,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,eAAe,EAAE,OAAO,CAAC,UAAU;SACtC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,6BAA6B,IAAI,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE;YACjG,OAAO,EAAE,kBAAkB;SAC9B,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAgC,CAAC;QAC3D,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,eAAe,CAAC,WAAW,CAAC;QACvE,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,gBAAgB,CAAC,WAAW,CAAC;QACzE,MAAM,MAAM,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;QACzD,OAAO;YACH,MAAM;YACN,WAAW,EAAE;gBACT,MAAM,EAAE,WAAW;gBACnB,MAAM,EAAE,YAAY;gBACpB,KAAK,EAAE,WAAW,IAAI,YAAY,CAAC,CAAC,CAAC,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS;aAC9E;SACU,CAAC;IAEpB,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,MAAsB,EAAE,MAA8C,EAAE,OAAyB;QAC3H,MAAM,QAAQ,GAAG,MAA6B,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE;YAC/B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,eAAe,EAAE,OAAO,CAAC,UAAU;SACtC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,6BAA6B,IAAI,CAAC,KAAK,CAAC,EAAE,iCAAiC,CAAC;QAEzF,MAAM,SAAS,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QAEpD,sEAAsE;QACtE,MAAM,SAAS,GAAG,MAA0B,CAAC;QAC7C,OAAQ,SAAiB,CAAC,SAAS,CAAC;QACpC,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QACpC,SAAS,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC;QAE5C,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE;YACpD,OAAO,EAAE,SAAS;YAClB,MAAM,EAAE,KAAK;SAChB,CAAC,CAAC;QAEH,OAAO,kBAAkB,CAAC,WAAW,EAAE,CAAC,IAAY,EAAE,EAAE;YACpD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,CAAC;YAC/D,OAAO,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;QAC3E,CAAC,CAAC,CAAA;IACN,CAAC;CAEJ"}
|
package/lib/types/index.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
export * from "./bedrock/index.js";
|
|
2
2
|
export * from "./huggingface_ie.js";
|
|
3
|
+
export * from "./mistral/index.js";
|
|
3
4
|
export * from "./openai.js";
|
|
4
5
|
export * from "./replicate.js";
|
|
5
|
-
export * from "./vertexai/index.js";
|
|
6
|
-
export * from "./togetherai/index.js";
|
|
7
6
|
export * from "./test/index.js";
|
|
7
|
+
export * from "./togetherai/index.js";
|
|
8
|
+
export * from "./vertexai/index.js";
|
|
8
9
|
//# sourceMappingURL=index.d.ts.map
|
package/lib/types/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { AIModel, AbstractDriver, Completion, DriverOptions, ExecutionOptions, PromptFormats, PromptSegment } from "@llumiverse/core";
|
|
2
|
+
import { FetchClient } from "api-fetch-client";
|
|
3
|
+
import { ResponseFormat } from "./types.js";
|
|
4
|
+
interface MistralAIDriverOptions extends DriverOptions {
|
|
5
|
+
apiKey: string;
|
|
6
|
+
endpoint_url?: string;
|
|
7
|
+
}
|
|
8
|
+
export declare class MistralAIDriver extends AbstractDriver<MistralAIDriverOptions, LLMMessage[]> {
|
|
9
|
+
provider: string;
|
|
10
|
+
apiKey: string;
|
|
11
|
+
defaultFormat: PromptFormats;
|
|
12
|
+
client: FetchClient;
|
|
13
|
+
endpointUrl?: string;
|
|
14
|
+
constructor(options: MistralAIDriverOptions);
|
|
15
|
+
getResponseFormat: (_options: ExecutionOptions) => ResponseFormat | undefined;
|
|
16
|
+
createPrompt(segments: PromptSegment[], opts: ExecutionOptions): LLMMessage[];
|
|
17
|
+
requestCompletion(messages: LLMMessage[], options: ExecutionOptions): Promise<Completion<any>>;
|
|
18
|
+
requestCompletionStream(messages: LLMMessage[], options: ExecutionOptions): Promise<AsyncIterable<string>>;
|
|
19
|
+
listModels(): Promise<AIModel<string>[]>;
|
|
20
|
+
listTrainableModels(): Promise<AIModel<string>[]>;
|
|
21
|
+
validateConnection(): Promise<boolean>;
|
|
22
|
+
generateEmbeddings(content: string, model?: string | undefined): Promise<{
|
|
23
|
+
embeddings: number[];
|
|
24
|
+
model: string;
|
|
25
|
+
}>;
|
|
26
|
+
}
|
|
27
|
+
interface LLMMessage {
|
|
28
|
+
role: string;
|
|
29
|
+
content: string;
|
|
30
|
+
}
|
|
31
|
+
export {};
|
|
32
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/mistral/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtI,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAA+C,cAAc,EAAE,MAAM,YAAY,CAAC;AAOzF,UAAU,sBAAuB,SAAQ,aAAa;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qBAAa,eAAgB,SAAQ,cAAc,CAAC,sBAAsB,EAAE,UAAU,EAAE,CAAC;IACrF,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,aAAa,CAAC;IAE7B,MAAM,EAAE,WAAW,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;gBAET,OAAO,EAAE,sBAAsB;IAW3C,iBAAiB,aAAc,gBAAgB,KAAG,cAAc,GAAG,SAAS,CAgB3E;IAED,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,gBAAgB,GAAG,UAAU,EAAE;IAiBvE,iBAAiB,CAAC,QAAQ,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAwB9F,uBAAuB,CAAC,QAAQ,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAqB1G,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;IAgB9C,mBAAmB,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;IAGjD,kBAAkB,IAAI,OAAO,CAAC,OAAO,CAAC;IAItC,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;KAAE,CAAC;CAIrH;AAED,UAAU,UAAU;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACnB"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
export interface ModelPermission {
|
|
2
|
+
id: string;
|
|
3
|
+
object: 'model_permission';
|
|
4
|
+
created: number;
|
|
5
|
+
allow_create_engine: boolean;
|
|
6
|
+
allow_sampling: boolean;
|
|
7
|
+
allow_logprobs: boolean;
|
|
8
|
+
allow_search_indices: boolean;
|
|
9
|
+
allow_view: boolean;
|
|
10
|
+
allow_fine_tuning: boolean;
|
|
11
|
+
organization: string;
|
|
12
|
+
group: string | null;
|
|
13
|
+
is_blocking: boolean;
|
|
14
|
+
}
|
|
15
|
+
export interface Model {
|
|
16
|
+
id: string;
|
|
17
|
+
object: 'model';
|
|
18
|
+
created: number;
|
|
19
|
+
owned_by: string;
|
|
20
|
+
root: string | null;
|
|
21
|
+
parent: string | null;
|
|
22
|
+
permission: ModelPermission[];
|
|
23
|
+
}
|
|
24
|
+
export interface ListModelsResponse {
|
|
25
|
+
object: 'list';
|
|
26
|
+
data: Model[];
|
|
27
|
+
}
|
|
28
|
+
export interface Function {
|
|
29
|
+
name: string;
|
|
30
|
+
description: string;
|
|
31
|
+
parameters: object;
|
|
32
|
+
}
|
|
33
|
+
export declare enum ToolType {
|
|
34
|
+
function = "function"
|
|
35
|
+
}
|
|
36
|
+
export interface FunctionCall {
|
|
37
|
+
name: string;
|
|
38
|
+
arguments: string;
|
|
39
|
+
}
|
|
40
|
+
export interface ToolCalls {
|
|
41
|
+
id: 'null';
|
|
42
|
+
type: ToolType;
|
|
43
|
+
function: FunctionCall;
|
|
44
|
+
}
|
|
45
|
+
export declare enum ResponseFormats {
|
|
46
|
+
text = "text",
|
|
47
|
+
json_object = "json_object"
|
|
48
|
+
}
|
|
49
|
+
export declare enum ToolChoice {
|
|
50
|
+
auto = "auto",
|
|
51
|
+
any = "any",
|
|
52
|
+
none = "none"
|
|
53
|
+
}
|
|
54
|
+
export interface ResponseFormat {
|
|
55
|
+
type: ResponseFormats;
|
|
56
|
+
}
|
|
57
|
+
export interface TokenUsage {
|
|
58
|
+
prompt_tokens: number;
|
|
59
|
+
completion_tokens: number;
|
|
60
|
+
total_tokens: number;
|
|
61
|
+
}
|
|
62
|
+
export interface ChatCompletionResponseChoice {
|
|
63
|
+
index: number;
|
|
64
|
+
message: {
|
|
65
|
+
role: string;
|
|
66
|
+
content: string;
|
|
67
|
+
};
|
|
68
|
+
finish_reason: string;
|
|
69
|
+
}
|
|
70
|
+
export interface ChatCompletionResponseChunkChoice {
|
|
71
|
+
index: number;
|
|
72
|
+
delta: {
|
|
73
|
+
role?: string;
|
|
74
|
+
content?: string;
|
|
75
|
+
tool_calls?: ToolCalls[];
|
|
76
|
+
};
|
|
77
|
+
finish_reason: string;
|
|
78
|
+
}
|
|
79
|
+
export interface ChatCompletionResponse {
|
|
80
|
+
id: string;
|
|
81
|
+
object: 'chat.completion';
|
|
82
|
+
created: number;
|
|
83
|
+
model: string;
|
|
84
|
+
choices: ChatCompletionResponseChoice[];
|
|
85
|
+
usage: TokenUsage;
|
|
86
|
+
}
|
|
87
|
+
export interface ChatCompletionResponseChunk {
|
|
88
|
+
id: string;
|
|
89
|
+
object: 'chat.completion.chunk';
|
|
90
|
+
created: number;
|
|
91
|
+
model: string;
|
|
92
|
+
choices: ChatCompletionResponseChunkChoice[];
|
|
93
|
+
}
|
|
94
|
+
export interface Embedding {
|
|
95
|
+
id: string;
|
|
96
|
+
object: 'embedding';
|
|
97
|
+
embedding: number[];
|
|
98
|
+
}
|
|
99
|
+
export interface EmbeddingResponse {
|
|
100
|
+
id: string;
|
|
101
|
+
object: 'list';
|
|
102
|
+
data: Embedding[];
|
|
103
|
+
model: string;
|
|
104
|
+
usage: TokenUsage;
|
|
105
|
+
}
|
|
106
|
+
export interface CompletionRequestParams {
|
|
107
|
+
model: string;
|
|
108
|
+
messages: Array<{
|
|
109
|
+
role: string;
|
|
110
|
+
name?: string;
|
|
111
|
+
content: string | string[];
|
|
112
|
+
tool_calls?: ToolCalls[];
|
|
113
|
+
}>;
|
|
114
|
+
tools?: Array<{
|
|
115
|
+
type: string;
|
|
116
|
+
function: Function;
|
|
117
|
+
}>;
|
|
118
|
+
temperature?: number;
|
|
119
|
+
maxTokens?: number;
|
|
120
|
+
topP?: number;
|
|
121
|
+
randomSeed?: number;
|
|
122
|
+
stream?: boolean;
|
|
123
|
+
/**
|
|
124
|
+
* @deprecated use safePrompt instead
|
|
125
|
+
*/
|
|
126
|
+
safeMode?: boolean;
|
|
127
|
+
safePrompt?: boolean;
|
|
128
|
+
toolChoice?: ToolChoice;
|
|
129
|
+
responseFormat?: ResponseFormat;
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/mistral/types.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,cAAc,EAAE,OAAO,CAAC;IACxB,cAAc,EAAE,OAAO,CAAC;IACxB,oBAAoB,EAAE,OAAO,CAAC;IAC9B,UAAU,EAAE,OAAO,CAAC;IACpB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,KAAK;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE,eAAe,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,kBAAkB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,KAAK,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,QAAQ;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,oBAAY,QAAQ;IAChB,QAAQ,aAAa;CACxB;AAED,MAAM,WAAW,YAAY;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,QAAQ,CAAC;IACf,QAAQ,EAAE,YAAY,CAAC;CAC1B;AAED,oBAAY,eAAe;IACvB,IAAI,SAAS;IACb,WAAW,gBAAgB;CAC9B;AAED,oBAAY,UAAU;IAClB,IAAI,SAAS;IACb,GAAG,QAAQ;IACX,IAAI,SAAS;CAChB;AAED,MAAM,WAAW,cAAc;IAC3B,IAAI,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,aAAa,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iCAAiC;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;KAC5B,CAAC;IACF,aAAa,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,sBAAsB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,iBAAiB,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,4BAA4B,EAAE,CAAC;IACxC,KAAK,EAAE,UAAU,CAAC;CACrB;AAED,MAAM,WAAW,2BAA2B;IACxC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,uBAAuB,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,iCAAiC,EAAE,CAAC;CAChD;AAED,MAAM,WAAW,SAAS;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,iBAAiB;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,UAAU,CAAC;CACrB;AAED,MAAM,WAAW,uBAAuB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;KAAE,CAAC,CAAC;IACxG,KAAK,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,QAAQ,CAAC;KAAE,CAAC,CAAC;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,cAAc,CAAC,EAAE,cAAc,CAAA;CAClC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/togetherai/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/togetherai/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEvH,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAG/C,UAAU,uBAAwB,SAAQ,aAAa;IACnD,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAiB,SAAQ,cAAc,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACjF,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,aAAa,CAAC;IAC7B,WAAW,EAAE,WAAW,CAAC;gBAEb,OAAO,EAAE,uBAAuB;IAU5C,iBAAiB,YAAa,gBAAgB;;;kBAM7C;IAEK,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IA2BtF,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAyBlG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;IAkB9C,mBAAmB,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;IAGjD,kBAAkB,IAAI,OAAO,CAAC,OAAO,CAAC;IAItC,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;KAAE,CAAC;CAIrH"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"palm-model-base.d.ts","sourceRoot":"","sources":["../../../../src/vertexai/models/palm-model-base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"palm-model-base.d.ts","sourceRoot":"","sources":["../../../../src/vertexai/models/palm-model-base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEvG,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAG3D,MAAM,WAAW,sBAAsB,CAAC,YAAY,GAAG,GAAG;IACtD,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,UAAU,EAAE,oBAAoB,CAAC;CACpC;AAED,MAAM,WAAW,mBAAmB,CAAC,SAAS,GAAG,GAAG;IAChD,MAAM,EAAE;QAAE,SAAS,EAAE,SAAS,CAAA;KAAE,EAAE,CAAC;IACnC,UAAU,EAAE;QACR,SAAS,EAAE;YACP,WAAW,CAAC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAA;aAAE,CAAC;YACnC,eAAe,CAAC,EAAE;gBAAE,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YAErC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,CAAA;SACjD,CAAA;KACJ,CAAA;CACJ;AAED,MAAM,WAAW,oBAAoB;IACjC,aAAa,EAAE;QACX,gBAAgB,EAAE;YACd,uBAAuB,EAAE,MAAM,CAAC;YAChC,WAAW,EAAE,MAAM,CAAA;SACtB,CAAC;QACF,eAAe,EAAE;YACb,uBAAuB,EAAE,MAAM,CAAC;YAChC,WAAW,EAAE,MAAM,CAAA;SACtB,CAAA;KACJ,CAAA;CACJ;AAGD,8BAAsB,2BAA2B,CAAC,mBAAmB,SAAS,sBAAsB,EAAE,gBAAgB,SAAS,mBAAmB,CAAE,YAAW,eAAe,CAAC,mBAAmB,GAAG,gBAAgB,CAAC;IAElN,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IAExB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IAE7B,QAAQ,CAAC,wBAAwB,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,aAAa,GAAG,mBAAmB;IAEjI,QAAQ,CAAC,0BAA0B,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM;IAE1D,QAAQ,CAAC,+BAA+B,CAAC,KAAK,EAAE,GAAG,GAAG,MAAM;IAE5D,YAAY,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,aAAa;IAIhF,iBAAiB,CAAC,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,mBAAmB,GAAG,gBAAgB,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IA0B9I,uBAAuB,CAAC,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,mBAAmB,GAAG,gBAAgB,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;CA6BnK"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@llumiverse/drivers",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "LLM driver implementations. Currently supported are: openai, huggingface, bedrock, replicate.",
|
|
6
6
|
"files": [
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"require": "./lib/cjs/index.js"
|
|
16
16
|
},
|
|
17
17
|
"scripts": {
|
|
18
|
-
"test": "vitest run",
|
|
18
|
+
"test": "vitest run --retry 3",
|
|
19
19
|
"build": "tsmod build",
|
|
20
20
|
"clean": "rm -rf ./lib tsconfig.tsbuildinfo"
|
|
21
21
|
},
|
|
@@ -63,7 +63,7 @@
|
|
|
63
63
|
"@google-cloud/aiplatform": "^3.10.0",
|
|
64
64
|
"@google-cloud/vertexai": "^0.2.1",
|
|
65
65
|
"@huggingface/inference": "^2.6.4",
|
|
66
|
-
"@llumiverse/core": "0.
|
|
66
|
+
"@llumiverse/core": "^0.9.0",
|
|
67
67
|
"api-fetch-client": "^0.8.6",
|
|
68
68
|
"eventsource": "^2.0.2",
|
|
69
69
|
"google-auth-library": "^9.6.1",
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
export * from "./bedrock/index.js";
|
|
2
2
|
export * from "./huggingface_ie.js";
|
|
3
|
+
export * from "./mistral/index.js";
|
|
3
4
|
export * from "./openai.js";
|
|
4
5
|
export * from "./replicate.js";
|
|
5
|
-
export * from "./vertexai/index.js";
|
|
6
|
-
export * from "./togetherai/index.js";
|
|
7
6
|
export * from "./test/index.js";
|
|
7
|
+
export * from "./togetherai/index.js";
|
|
8
|
+
export * from "./vertexai/index.js";
|
|
9
|
+
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { AIModel, AbstractDriver, Completion, DriverOptions, ExecutionOptions, PromptFormats, PromptSegment } from "@llumiverse/core";
|
|
2
|
+
import { transformSSEStream } from "@llumiverse/core/async";
|
|
3
|
+
import { FetchClient } from "api-fetch-client";
|
|
4
|
+
import { CompletionRequestParams, ListModelsResponse, ResponseFormat } from "./types.js";
|
|
5
|
+
|
|
6
|
+
//TODO retry on 429
|
|
7
|
+
//const RETRY_STATUS_CODES = [429, 500, 502, 503, 504];
|
|
8
|
+
|
|
9
|
+
const ENDPOINT = 'https://api.mistral.ai';
|
|
10
|
+
|
|
11
|
+
interface MistralAIDriverOptions extends DriverOptions {
|
|
12
|
+
apiKey: string;
|
|
13
|
+
endpoint_url?: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export class MistralAIDriver extends AbstractDriver<MistralAIDriverOptions, LLMMessage[]> {
|
|
17
|
+
provider: string;
|
|
18
|
+
apiKey: string;
|
|
19
|
+
defaultFormat: PromptFormats;
|
|
20
|
+
//client: MistralClient;
|
|
21
|
+
client: FetchClient;
|
|
22
|
+
endpointUrl?: string;
|
|
23
|
+
|
|
24
|
+
constructor(options: MistralAIDriverOptions) {
|
|
25
|
+
super(options);
|
|
26
|
+
this.provider = "MistralAI";
|
|
27
|
+
this.defaultFormat = PromptFormats.genericTextLLM;
|
|
28
|
+
this.apiKey = options.apiKey;
|
|
29
|
+
//this.client = new MistralClient(options.apiKey, options.endpointUrl);
|
|
30
|
+
this.client = new FetchClient(options.endpoint_url || ENDPOINT).withHeaders({
|
|
31
|
+
authorization: `Bearer ${this.apiKey}`
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
getResponseFormat = (_options: ExecutionOptions): ResponseFormat | undefined => {
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
/*const responseFormatJson: ResponseFormat = {
|
|
39
|
+
type: "json_object",
|
|
40
|
+
} as ResponseFormat
|
|
41
|
+
|
|
42
|
+
const responseFormatText: ResponseFormat = {
|
|
43
|
+
type: "text",
|
|
44
|
+
} as ResponseFormat;
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
//return _options.resultSchema ? responseFormatJson : responseFormatText;
|
|
48
|
+
|
|
49
|
+
//TODO remove this when Mistral properly supports the parameters - it makes an error for now
|
|
50
|
+
return undefined
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
createPrompt(segments: PromptSegment[], opts: ExecutionOptions): LLMMessage[] {
|
|
54
|
+
// use same format as OpenAI as that's what MistralAI uses
|
|
55
|
+
const prompts = super.createPrompt(segments, { ...opts, format: PromptFormats.openai })
|
|
56
|
+
|
|
57
|
+
//Add JSON instruction is schema is provided
|
|
58
|
+
if (opts.resultSchema) {
|
|
59
|
+
const content = "The user is explicitely instructing that the result should be a JSON object.\nThe schema is as follows: \n" + JSON.stringify(opts.resultSchema);
|
|
60
|
+
prompts.push({
|
|
61
|
+
role: "user",
|
|
62
|
+
content: content
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return prompts;
|
|
67
|
+
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async requestCompletion(messages: LLMMessage[], options: ExecutionOptions): Promise<Completion<any>> {
|
|
71
|
+
|
|
72
|
+
const res = await this.client.post('/v1/chat/completions', {
|
|
73
|
+
payload: _makeChatCompletionRequest({
|
|
74
|
+
model: options.model,
|
|
75
|
+
messages: messages,
|
|
76
|
+
maxTokens: options.max_tokens ?? 1024,
|
|
77
|
+
temperature: options.temperature ?? 0.7,
|
|
78
|
+
responseFormat: this.getResponseFormat(options),
|
|
79
|
+
})
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
const result = res.choices[0]?.message.content;
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
result: result,
|
|
86
|
+
token_usage: {
|
|
87
|
+
prompt: res.usage.prompt_tokens,
|
|
88
|
+
result: res.usage.completion_tokens,
|
|
89
|
+
total: res.usage.total_tokens,
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async requestCompletionStream(messages: LLMMessage[], options: ExecutionOptions): Promise<AsyncIterable<string>> {
|
|
95
|
+
|
|
96
|
+
const stream = await this.client.post('/v1/chat/completions', {
|
|
97
|
+
payload: _makeChatCompletionRequest({
|
|
98
|
+
model: options.model,
|
|
99
|
+
messages: messages,
|
|
100
|
+
maxTokens: options.max_tokens ?? 1024,
|
|
101
|
+
temperature: options.temperature ?? 0.7,
|
|
102
|
+
responseFormat: this.getResponseFormat(options),
|
|
103
|
+
stream: true
|
|
104
|
+
}),
|
|
105
|
+
reader: 'sse'
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
return transformSSEStream(stream, (data: string) => {
|
|
109
|
+
const json = JSON.parse(data);
|
|
110
|
+
return json.choices[0]?.delta.content ?? '';
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
async listModels(): Promise<AIModel<string>[]> {
|
|
116
|
+
const models: ListModelsResponse = await this.client.get('v1/models');
|
|
117
|
+
|
|
118
|
+
const aimodels = models.data.map(m => {
|
|
119
|
+
return {
|
|
120
|
+
id: m.id,
|
|
121
|
+
name: m.id,
|
|
122
|
+
description: undefined,
|
|
123
|
+
provider: m.owned_by,
|
|
124
|
+
formats: [PromptFormats.genericTextLLM],
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
return aimodels;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
listTrainableModels(): Promise<AIModel<string>[]> {
|
|
132
|
+
throw new Error("Method not implemented.");
|
|
133
|
+
}
|
|
134
|
+
validateConnection(): Promise<boolean> {
|
|
135
|
+
throw new Error("Method not implemented.");
|
|
136
|
+
}
|
|
137
|
+
//@ts-ignore
|
|
138
|
+
generateEmbeddings(content: string, model?: string | undefined): Promise<{ embeddings: number[]; model: string; }> {
|
|
139
|
+
throw new Error("Method not implemented.");
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
interface LLMMessage {
|
|
145
|
+
role: string;
|
|
146
|
+
content: string;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Creates a chat completion request
|
|
152
|
+
* @param {*} model
|
|
153
|
+
* @param {*} messages
|
|
154
|
+
* @param {*} tools
|
|
155
|
+
* @param {*} temperature
|
|
156
|
+
* @param {*} maxTokens
|
|
157
|
+
* @param {*} topP
|
|
158
|
+
* @param {*} randomSeed
|
|
159
|
+
* @param {*} stream
|
|
160
|
+
* @param {*} safeMode deprecated use safePrompt instead
|
|
161
|
+
* @param {*} safePrompt
|
|
162
|
+
* @param {*} toolChoice
|
|
163
|
+
* @param {*} responseFormat
|
|
164
|
+
* @return {Promise<Object>}
|
|
165
|
+
*/
|
|
166
|
+
function _makeChatCompletionRequest({
|
|
167
|
+
model,
|
|
168
|
+
messages,
|
|
169
|
+
tools,
|
|
170
|
+
temperature,
|
|
171
|
+
maxTokens,
|
|
172
|
+
topP,
|
|
173
|
+
randomSeed,
|
|
174
|
+
stream,
|
|
175
|
+
safeMode,
|
|
176
|
+
safePrompt,
|
|
177
|
+
toolChoice,
|
|
178
|
+
responseFormat,
|
|
179
|
+
}: CompletionRequestParams) {
|
|
180
|
+
return {
|
|
181
|
+
model: model,
|
|
182
|
+
messages: messages,
|
|
183
|
+
tools: tools ?? undefined,
|
|
184
|
+
temperature: temperature ?? undefined,
|
|
185
|
+
max_tokens: maxTokens ?? undefined,
|
|
186
|
+
top_p: topP ?? undefined,
|
|
187
|
+
random_seed: randomSeed ?? undefined,
|
|
188
|
+
stream: stream ?? undefined,
|
|
189
|
+
safe_prompt: (safeMode || safePrompt) ?? undefined,
|
|
190
|
+
tool_choice: toolChoice ?? undefined,
|
|
191
|
+
response_format: responseFormat ?? undefined,
|
|
192
|
+
};
|
|
193
|
+
};
|