modelfusion 0.116.1 → 0.118.0

package/CHANGELOG.md

@@ -1,5 +1,72 @@
 # Changelog
 
+## v0.118.0 - 2024-01-07
+
+### Added
+
+- `LlamaCppCompletionModel` supports setting the prompt template in the settings. Prompt formats are available under `llamacpp.prompt.*`. You can then call `.withTextPrompt()`, `.withInstructionPrompt()` or `.withChatPrompt()` to use a standardized prompt.
+
+  ```ts
+  const model = llamacpp
+    .CompletionTextGenerator({
+      // run https://huggingface.co/TheBloke/OpenHermes-2.5-Mistral-7B-GGUF with llama.cpp
+      promptTemplate: llamacpp.prompt.ChatML,
+      contextWindowSize: 4096,
+      maxGenerationTokens: 512,
+    })
+    .withChatPrompt();
+  ```
+
+### Changed
+
+- **breaking change**: renamed `response` to `rawResponse` when using `fullResponse: true` setting.
+- **breaking change**: renamed `llamacpp.TextGenerator` to `llamacpp.CompletionTextGenerator`.
+
+### Removed
+
+- **breaking change**: removed `.withTextPromptTemplate` on `LlamaCppCompletionModel`.
+
+## v0.117.0 - 2024-01-06
+
+### Added
+
+- Predefined Llama.cpp GBNF grammars:
+
+  - `llamacpp.grammar.json`: Restricts the output to JSON.
+  - `llamacpp.grammar.jsonArray`: Restricts the output to a JSON array.
+  - `llamacpp.grammar.list`: Restricts the output to a newline-separated list where each line starts with `- `.
+
+- Llama.cpp structure generation support:
+
+  ```ts
+  const structure = await generateStructure(
+    llamacpp
+      .TextGenerator({
+        // run openhermes-2.5-mistral-7b.Q4_K_M.gguf in llama.cpp
+        maxGenerationTokens: 1024,
+        temperature: 0,
+      })
+      .withTextPromptTemplate(ChatMLPrompt.instruction()) // needed for jsonStructurePrompt.text()
+      .asStructureGenerationModel(jsonStructurePrompt.text()), // automatically restrict the output to JSON
+
+    zodSchema(
+      z.object({
+        characters: z.array(
+          z.object({
+            name: z.string(),
+            class: z
+              .string()
+              .describe("Character class, e.g. warrior, mage, or thief."),
+            description: z.string(),
+          })
+        ),
+      })
+    ),
+
+    "Generate 3 character descriptions for a fantasy role playing game. "
+  );
+  ```
+
 ## v0.116.0 - 2024-01-05
 
 ### Added

package/README.md

@@ -152,7 +152,7 @@ const sentiment = await generateStructure(
 );
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Ollama](https://modelfusion.dev//integration/model-provider/ollama)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Ollama](https://modelfusion.dev//integration/model-provider/ollama), [Llama.cpp](https://modelfusion.dev//integration/model-provider/llama.cpp)
 
 #### streamStructure
 
@@ -198,7 +198,7 @@ for await (const part of structureStream) {
 }
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Ollama](https://modelfusion.dev//integration/model-provider/ollama)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Ollama](https://modelfusion.dev//integration/model-provider/ollama), [Llama.cpp](https://modelfusion.dev//integration/model-provider/llama.cpp)
 
 ### [Generate Image](https://modelfusion.dev/guide/function/generate-image)
 
@@ -424,14 +424,15 @@ const text = await generateText(
 #### Instruction Prompt Example
 
 ```ts
-// example assumes you are running https://huggingface.co/TheBloke/Llama-2-7B-GGUF with llama.cpp
 const text = await generateText(
   llamacpp
-    .TextGenerator({
+    .CompletionTextGenerator({
+      // run https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF with llama.cpp
+      promptTemplate: llamacpp.prompt.Llama2, // Set prompt template
       contextWindowSize: 4096, // Llama 2 context window size
-      maxGenerationTokens: 1000,
+      maxGenerationTokens: 512,
     })
-    .withTextPromptTemplate(Llama2Prompt.instruction()),
+    .withInstructionPrompt(),
   {
     system: "You are a story writer.",
     instruction: "Write a short story about a robot learning to love.",
@@ -503,11 +504,11 @@ const image = await generateImage(
 
 ### Metadata and original responses
 
-ModelFusion model functions return rich responses that include the original response and metadata when you set the `fullResponse` option to `true`.
+ModelFusion model functions return rich responses that include the raw (original) response and metadata when you set the `fullResponse` option to `true`.
 
 ```ts
-// access the full response (needs to be typed) and the metadata:
-const { text, response, metadata } = await generateText(
+// access the raw response (needs to be typed) and the metadata:
+const { text, rawResponse, metadata } = await generateText(
   openai.CompletionTextGenerator({
     model: "gpt-3.5-turbo-instruct",
     maxGenerationTokens: 1000,
@@ -519,8 +520,8 @@ const { text, response, metadata } = await generateText(
 
 console.log(metadata);
 
-// cast to the response type:
-for (const choice of (response as OpenAICompletionResponse).choices) {
+// cast to the raw response type:
+for (const choice of (rawResponse as OpenAICompletionResponse).choices) {
   console.log(choice.text);
 }
 ```
@@ -563,6 +564,8 @@ modelfusion.setLogFormat("detailed-object"); // log full events
   - [Split Text](https://modelfusion.dev/guide/text-chunk/split)
 - [Utilities](https://modelfusion.dev/guide/util/)
   - [API Configuration](https://modelfusion.dev/guide/util/api-configuration)
+    - [Base URL](https://modelfusion.dev/guide/util/api-configuration/base-url)
+    - [Headers](https://modelfusion.dev/guide/util/api-configuration/headers)
     - [Retry strategies](https://modelfusion.dev/guide/util/api-configuration/retry)
     - [Throttling strategies](https://modelfusion.dev/guide/util/api-configuration/throttle)
   - [Logging](https://modelfusion.dev/guide/util/logging)

package/core/getFunctionCallLogger.cjs

@@ -36,11 +36,11 @@ const detailedObjectObserver = {
         // Remove the "response" property from the result (if any):
         if (event.eventType === "finished" &&
             event.result != null &&
-            "response" in event.result &&
-            event.result?.response != null) {
+            "rawResponse" in event.result &&
+            event.result?.rawResponse != null) {
             event = {
                 ...event,
-                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "response")
+                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "rawResponse")
                 // eslint-disable-next-line @typescript-eslint/no-explicit-any
                 ),
             };
@@ -73,11 +73,11 @@ const detailedJsonObserver = {
         // Remove the "response" property from the result (if any):
         if (event.eventType === "finished" &&
             event.result != null &&
-            "response" in event.result &&
-            event.result?.response != null) {
+            "rawResponse" in event.result &&
+            event.result?.rawResponse != null) {
             event = {
                 ...event,
-                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "response")
+                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "rawResponse")
                 // eslint-disable-next-line @typescript-eslint/no-explicit-any
                 ),
             };

package/core/getFunctionCallLogger.js

@@ -32,11 +32,11 @@ const detailedObjectObserver = {
         // Remove the "response" property from the result (if any):
         if (event.eventType === "finished" &&
             event.result != null &&
-            "response" in event.result &&
-            event.result?.response != null) {
+            "rawResponse" in event.result &&
+            event.result?.rawResponse != null) {
             event = {
                 ...event,
-                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "response")
+                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "rawResponse")
                 // eslint-disable-next-line @typescript-eslint/no-explicit-any
                 ),
             };
@@ -69,11 +69,11 @@ const detailedJsonObserver = {
         // Remove the "response" property from the result (if any):
         if (event.eventType === "finished" &&
             event.result != null &&
-            "response" in event.result &&
-            event.result?.response != null) {
+            "rawResponse" in event.result &&
+            event.result?.rawResponse != null) {
             event = {
                 ...event,
-                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "response")
+                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "rawResponse")
                 // eslint-disable-next-line @typescript-eslint/no-explicit-any
                 ),
             };

package/model-function/ModelCallEvent.d.ts

@@ -22,7 +22,7 @@ export interface BaseModelCallStartedEvent extends BaseFunctionStartedEvent {
 }
 export type BaseModelCallFinishedEventResult = {
     status: "success";
-    response: unknown;
+    rawResponse: unknown;
     value: unknown;
     /**
      * Optional usage information for the model call. The type depends on the call type.

package/model-function/embed/EmbeddingEvent.d.ts

@@ -6,7 +6,7 @@ export interface EmbeddingStartedEvent extends BaseModelCallStartedEvent {
 }
 export type EmbeddingFinishedEventResult = {
     status: "success";
-    response: unknown;
+    rawResponse: unknown;
     value: Vector | Array<Vector>;
 } | {
     status: "error";

package/model-function/embed/EmbeddingModel.d.ts

@@ -17,7 +17,7 @@ export interface EmbeddingModel<VALUE, SETTINGS extends EmbeddingModelSettings =
      */
     readonly isParallelizable: boolean;
     doEmbedValues(values: VALUE[], options: FunctionCallOptions): PromiseLike<{
-        response: unknown;
+        rawResponse: unknown;
         embeddings: Vector[];
     }>;
 }

package/model-function/embed/embed.cjs

@@ -32,13 +32,13 @@ async function embedMany(model, values, options) {
                     responses.push(response);
                 }
             }
-            const rawResponses = responses.map((response) => response.response);
+            const rawResponses = responses.map((response) => response.rawResponse);
             const embeddings = [];
             for (const response of responses) {
                 embeddings.push(...response.embeddings);
             }
             return {
-                response: rawResponses,
+                rawResponse: rawResponses,
                 extractedValue: embeddings,
             };
         },
@@ -46,7 +46,7 @@ async function embedMany(model, values, options) {
     return options?.fullResponse
         ? {
             embeddings: fullResponse.value,
-            response: fullResponse.response,
+            rawResponse: fullResponse.rawResponse,
             metadata: fullResponse.metadata,
         }
         : fullResponse.value;
@@ -61,7 +61,7 @@ async function embed(model, value, options) {
         generateResponse: async (options) => {
             const result = await model.doEmbedValues([value], options);
             return {
-                response: result.response,
+                rawResponse: result.rawResponse,
                 extractedValue: result.embeddings[0],
             };
         },
@@ -69,7 +69,7 @@ async function embed(model, value, options) {
     return options?.fullResponse
         ? {
             embedding: fullResponse.value,
-            response: fullResponse.response,
+            rawResponse: fullResponse.rawResponse,
             metadata: fullResponse.metadata,
         }
         : fullResponse.value;

package/model-function/embed/embed.d.ts

@@ -29,7 +29,7 @@ export declare function embedMany<VALUE>(model: EmbeddingModel<VALUE, EmbeddingM
     fullResponse: true;
 }): Promise<{
     embeddings: Vector[];
-    response: unknown;
+    rawResponse: unknown;
     metadata: ModelCallMetadata;
 }>;
 /**
@@ -56,6 +56,6 @@ export declare function embed<VALUE>(model: EmbeddingModel<VALUE, EmbeddingModel
     fullResponse: true;
 }): Promise<{
     embedding: Vector;
-    response: unknown;
+    rawResponse: unknown;
     metadata: ModelCallMetadata;
 }>;

package/model-function/embed/embed.js

@@ -29,13 +29,13 @@ export async function embedMany(model, values, options) {
                     responses.push(response);
                 }
             }
-            const rawResponses = responses.map((response) => response.response);
+            const rawResponses = responses.map((response) => response.rawResponse);
             const embeddings = [];
             for (const response of responses) {
                 embeddings.push(...response.embeddings);
             }
             return {
-                response: rawResponses,
+                rawResponse: rawResponses,
                 extractedValue: embeddings,
             };
         },
@@ -43,7 +43,7 @@ export async function embedMany(model, values, options) {
     return options?.fullResponse
         ? {
             embeddings: fullResponse.value,
-            response: fullResponse.response,
+            rawResponse: fullResponse.rawResponse,
             metadata: fullResponse.metadata,
         }
         : fullResponse.value;
@@ -57,7 +57,7 @@ export async function embed(model, value, options) {
         generateResponse: async (options) => {
             const result = await model.doEmbedValues([value], options);
             return {
-                response: result.response,
+                rawResponse: result.rawResponse,
                 extractedValue: result.embeddings[0],
             };
         },
@@ -65,7 +65,7 @@ export async function embed(model, value, options) {
     return options?.fullResponse
         ? {
             embedding: fullResponse.value,
-            response: fullResponse.response,
+            rawResponse: fullResponse.rawResponse,
             metadata: fullResponse.metadata,
         }
         : fullResponse.value;

package/model-function/executeStandardCall.cjs

@@ -78,7 +78,7 @@ async function executeStandardCall({ model, options, input, functionType, genera
         });
         throw result.error;
     }
-    const response = result.value.response;
+    const rawResponse = result.value.rawResponse;
     const value = result.value.extractedValue;
     const usage = result.value.usage;
     eventSource.notify({
@@ -87,13 +87,13 @@ async function executeStandardCall({ model, options, input, functionType, genera
         result: {
             status: "success",
             usage,
-            response,
+            rawResponse,
             value,
         },
     });
     return {
         value,
-        response,
+        rawResponse,
         metadata: {
             model: model.modelInformation,
             callId: finishMetadata.callId,

package/model-function/executeStandardCall.d.ts

@@ -8,12 +8,12 @@ export declare function executeStandardCall<VALUE, MODEL extends Model<ModelSett
     input: unknown;
     functionType: ModelCallStartedEvent["functionType"];
     generateResponse: (options: FunctionCallOptions) => PromiseLike<{
-        response: unknown;
+        rawResponse: unknown;
         extractedValue: VALUE;
         usage?: unknown;
     }>;
 }): Promise<{
     value: VALUE;
-    response: unknown;
+    rawResponse: unknown;
     metadata: ModelCallMetadata;
 }>;

package/model-function/executeStandardCall.js

@@ -75,7 +75,7 @@ export async function executeStandardCall({ model, options, input, functionType,
         });
         throw result.error;
     }
-    const response = result.value.response;
+    const rawResponse = result.value.rawResponse;
     const value = result.value.extractedValue;
     const usage = result.value.usage;
     eventSource.notify({
@@ -84,13 +84,13 @@ export async function executeStandardCall({ model, options, input, functionType,
         result: {
             status: "success",
             usage,
-            response,
+            rawResponse,
             value,
         },
     });
     return {
         value,
-        response,
+        rawResponse,
         metadata: {
             model: model.modelInformation,
             callId: finishMetadata.callId,

package/model-function/generate-image/ImageGenerationEvent.d.ts

@@ -4,7 +4,7 @@ export interface ImageGenerationStartedEvent extends BaseModelCallStartedEvent {
 }
 export type ImageGenerationFinishedEventResult = {
     status: "success";
-    response: unknown;
+    rawResponse: unknown;
     value: string;
 } | {
     status: "error";

package/model-function/generate-image/ImageGenerationModel.d.ts

@@ -15,7 +15,7 @@ export interface ImageGenerationModelSettings extends ModelSettings {
 }
 export interface ImageGenerationModel<PROMPT, SETTINGS extends ImageGenerationModelSettings = ImageGenerationModelSettings> extends Model<SETTINGS> {
     doGenerateImages(prompt: PROMPT, options: FunctionCallOptions): PromiseLike<{
-        response: unknown;
+        rawResponse: unknown;
         base64Images: string[];
     }>;
     withPromptTemplate<INPUT_PROMPT>(promptTemplate: PromptTemplate<INPUT_PROMPT, PROMPT>): ImageGenerationModel<INPUT_PROMPT, SETTINGS>;

package/model-function/generate-image/PromptTemplateImageGenerationModel.d.ts

@@ -11,7 +11,7 @@ export declare class PromptTemplateImageGenerationModel<PROMPT, MODEL_PROMPT, SE
     get modelInformation(): import("../ModelInformation.js").ModelInformation;
     get settings(): SETTINGS;
     doGenerateImages(prompt: PROMPT, options: FunctionCallOptions): PromiseLike<{
-        response: unknown;
+        rawResponse: unknown;
         base64Images: string[];
     }>;
     get settingsForEvent(): Partial<SETTINGS>;

package/model-function/generate-image/generateImage.cjs

@@ -11,7 +11,7 @@ async function generateImage(model, prompt, options) {
         generateResponse: async (options) => {
             const result = await model.doGenerateImages(prompt, options);
             return {
-                response: result.response,
+                rawResponse: result.rawResponse,
                 extractedValue: result.base64Images,
             };
         },
@@ -24,7 +24,7 @@ async function generateImage(model, prompt, options) {
             imageBase64: imagesBase64[0],
             images,
             imagesBase64,
-            response: fullResponse.response,
+            rawResponse: fullResponse.rawResponse,
             metadata: fullResponse.metadata,
         }
         : images[0];

package/model-function/generate-image/generateImage.d.ts

@@ -36,6 +36,6 @@ export declare function generateImage<PROMPT>(model: ImageGenerationModel<PROMPT
     imageBase64: string;
     images: Buffer[];
     imagesBase64: string[];
-    response: unknown;
+    rawResponse: unknown;
     metadata: ModelCallMetadata;
 }>;

package/model-function/generate-image/generateImage.js

@@ -8,7 +8,7 @@ export async function generateImage(model, prompt, options) {
         generateResponse: async (options) => {
             const result = await model.doGenerateImages(prompt, options);
             return {
-                response: result.response,
+                rawResponse: result.rawResponse,
                 extractedValue: result.base64Images,
             };
         },
@@ -21,7 +21,7 @@ export async function generateImage(model, prompt, options) {
             imageBase64: imagesBase64[0],
             images,
             imagesBase64,
-            response: fullResponse.response,
+            rawResponse: fullResponse.rawResponse,
             metadata: fullResponse.metadata,
         }
         : images[0];

package/model-function/generate-speech/SpeechGenerationEvent.d.ts

@@ -6,7 +6,7 @@ export interface SpeechGenerationStartedEvent extends BaseModelCallStartedEvent
 }
 export type SpeechGenerationFinishedEventResult = {
     status: "success";
-    response: unknown;
+    rawResponse: unknown;
     value: Buffer;
 } | {
     status: "error";

package/model-function/generate-speech/generateSpeech.cjs

@@ -11,7 +11,7 @@ async function generateSpeech(model, text, options) {
         generateResponse: async (options) => {
             const response = await model.doGenerateSpeechStandard(text, options);
             return {
-                response,
+                rawResponse: response,
                 extractedValue: response,
             };
         },
@@ -19,7 +19,7 @@ async function generateSpeech(model, text, options) {
     return options?.fullResponse
         ? {
             speech: fullResponse.value,
-            response: fullResponse.response,
+            rawResponse: fullResponse.rawResponse,
             metadata: fullResponse.metadata,
         }
         : fullResponse.value;

package/model-function/generate-speech/generateSpeech.d.ts

@@ -27,6 +27,6 @@ export declare function generateSpeech(model: SpeechGenerationModel<SpeechGenera
     fullResponse: true;
 }): Promise<{
     speech: Buffer;
-    response: unknown;
+    rawResponse: unknown;
     metadata: ModelCallMetadata;
 }>;

package/model-function/generate-speech/generateSpeech.js

@@ -8,7 +8,7 @@ export async function generateSpeech(model, text, options) {
         generateResponse: async (options) => {
             const response = await model.doGenerateSpeechStandard(text, options);
             return {
-                response,
+                rawResponse: response,
                 extractedValue: response,
             };
         },
@@ -16,7 +16,7 @@ export async function generateSpeech(model, text, options) {
     return options?.fullResponse
         ? {
             speech: fullResponse.value,
-            response: fullResponse.response,
+            rawResponse: fullResponse.rawResponse,
             metadata: fullResponse.metadata,
         }
         : fullResponse.value;

package/model-function/generate-structure/StructureFromTextGenerationModel.cjs

@@ -30,7 +30,7 @@ class StructureFromTextGenerationModel {
         return this.model.settingsForEvent;
     }
     async doGenerateStructure(schema, prompt, options) {
-        const { response, text } = await (0, generateText_js_1.generateText)(this.model, this.template.createPrompt(prompt, schema), {
+        const { rawResponse: response, text } = await (0, generateText_js_1.generateText)(this.model, this.template.createPrompt(prompt, schema), {
             ...options,
             fullResponse: true,
         });

package/model-function/generate-structure/StructureFromTextGenerationModel.js

@@ -27,7 +27,7 @@ export class StructureFromTextGenerationModel {
         return this.model.settingsForEvent;
     }
     async doGenerateStructure(schema, prompt, options) {
-        const { response, text } = await generateText(this.model, this.template.createPrompt(prompt, schema), {
+        const { rawResponse: response, text } = await generateText(this.model, this.template.createPrompt(prompt, schema), {
             ...options,
             fullResponse: true,
         });

package/model-function/generate-structure/StructureFromTextStreamingModel.cjs

@@ -12,7 +12,7 @@ class StructureFromTextStreamingModel extends StructureFromTextGenerationModel_j
         super(options);
     }
     async doGenerateStructure(schema, prompt, options) {
-        const { response, text } = await (0, generateText_js_1.generateText)(this.model, this.template.createPrompt(prompt, schema), {
+        const { rawResponse: response, text } = await (0, generateText_js_1.generateText)(this.model, this.template.createPrompt(prompt, schema), {
             ...options,
             fullResponse: true,
         });

package/model-function/generate-structure/StructureFromTextStreamingModel.js

@@ -9,7 +9,7 @@ export class StructureFromTextStreamingModel extends StructureFromTextGeneration
         super(options);
     }
     async doGenerateStructure(schema, prompt, options) {
-        const { response, text } = await generateText(this.model, this.template.createPrompt(prompt, schema), {
+        const { rawResponse: response, text } = await generateText(this.model, this.template.createPrompt(prompt, schema), {
             ...options,
             fullResponse: true,
         });

package/model-function/generate-structure/StructureGenerationEvent.d.ts

@@ -4,7 +4,7 @@ export interface StructureGenerationStartedEvent extends BaseModelCallStartedEve
 }
 export type StructureGenerationFinishedEventResult = {
     status: "success";
-    response: unknown;
+    rawResponse: unknown;
     value: unknown;
     usage?: {
         promptTokens: number;

package/model-function/generate-structure/generateStructure.cjs

@@ -29,7 +29,7 @@ async function generateStructure(model, schema, prompt, options) {
             }
             const value = parseResult.data;
             return {
-                response: result.response,
+                rawResponse: result.response,
                 extractedValue: value,
                 usage: result.usage,
             };
@@ -38,7 +38,7 @@ async function generateStructure(model, schema, prompt, options) {
     return options?.fullResponse
         ? {
             structure: fullResponse.value,
-            response: fullResponse.response,
+            rawResponse: fullResponse.rawResponse,
             metadata: fullResponse.metadata,
         }
         : fullResponse.value;

package/model-function/generate-structure/generateStructure.d.ts

@@ -44,6 +44,6 @@ export declare function generateStructure<STRUCTURE, PROMPT, SETTINGS extends St
     fullResponse: true;
 }): Promise<{
     structure: STRUCTURE;
-    response: unknown;
+    rawResponse: unknown;
     metadata: ModelCallMetadata;
 }>;

package/model-function/generate-structure/generateStructure.js

@@ -26,7 +26,7 @@ export async function generateStructure(model, schema, prompt, options) {
             }
             const value = parseResult.data;
             return {
-                response: result.response,
+                rawResponse: result.response,
                 extractedValue: value,
                 usage: result.usage,
             };
@@ -35,7 +35,7 @@ export async function generateStructure(model, schema, prompt, options) {
     return options?.fullResponse
         ? {
             structure: fullResponse.value,
-            response: fullResponse.response,
+            rawResponse: fullResponse.rawResponse,
             metadata: fullResponse.metadata,
         }
         : fullResponse.value;