modelfusion 0.107.0 → 0.109.0

package/CHANGELOG.md

@@ -1,5 +1,49 @@
 # Changelog
 
+## v0.109.0 - 2023-12-30
+
+### Added
+
+- [Open AI compatible completion model](https://modelfusion.dev/integration/model-provider/openaicompatible/). It e.g. works with Fireworks AI.
+- Together AI API configuration (for Open AI compatible chat models):
+
+  ```ts
+  import {
+    TogetherAIApiConfiguration,
+    openaicompatible,
+    streamText,
+  } from "modelfusion";
+
+  const textStream = await streamText(
+    openaicompatible
+      .ChatTextGenerator({
+        api: new TogetherAIApiConfiguration(),
+        model: "mistralai/Mixtral-8x7B-Instruct-v0.1",
+      })
+      .withTextPrompt(),
+
+    "Write a story about a robot learning to love"
+  );
+  ```
+
+- Updated Llama.cpp model settings. GBNF grammars can be passed into the `grammar` setting:
+
+  ```ts
+  const text = await generateText(
+    llamacpp
+      .TextGenerator({
+        maxGenerationTokens: 512,
+        temperature: 0,
+        // simple list grammar:
+        grammar: `root ::= ("- " item)+
+  item ::= [^\\n]+ "\\n"`,
+      })
+      .withTextPromptTemplate(MistralInstructPrompt.text()),
+
+    "List 5 ingredients for a lasagna:\n\n"
+  );
+  ```
+
 ## v0.107.0 - 2023-12-29
 
 ### Added

package/README.md

@@ -538,16 +538,17 @@ const textStream = await streamText(
 );
 ```
 
-| Prompt Template | Text Prompt | Instruction Prompt | Chat Prompt |
-| --------------- | ----------- | ------------------ | ----------- |
-| OpenAI Chat     | ✅          | ✅                 | ✅          |
-| Anthropic       | ✅          | ✅                 | ✅          |
-| Llama 2         | ✅          | ✅                 | ✅          |
-| ChatML          | ✅          | ✅                 | ✅          |
-| NeuralChat      | ✅          | ✅                 | ✅          |
-| Alpaca          | ✅          | ✅                 | ❌          |
-| Vicuna          | ❌          | ❌                 | ✅          |
-| Generic Text    | ✅          | ✅                 | ✅          |
+| Prompt Template  | Text Prompt | Instruction Prompt | Chat Prompt |
+| ---------------- | ----------- | ------------------ | ----------- |
+| OpenAI Chat      | ✅          | ✅                 | ✅          |
+| Anthropic        | ✅          | ✅                 | ✅          |
+| Llama 2          | ✅          | ✅                 | ✅          |
+| ChatML           | ✅          | ✅                 | ✅          |
+| NeuralChat       | ✅          | ✅                 | ✅          |
+| Mistral Instruct | ✅          | ✅                 | ✅          |
+| Alpaca           | ✅          | ✅                 | ❌          |
+| Vicuna           | ❌          | ❌                 | ✅          |
+| Generic Text     | ✅          | ✅                 | ✅          |
 
 ### [Image Generation Prompt Templates](https://modelfusion.dev/guide/function/generate-image/prompt-format)
 

package/model-provider/llamacpp/LlamaCppCompletionModel.cjs

@@ -56,10 +56,10 @@ class LlamaCppCompletionModel extends AbstractModel_js_1.AbstractModel {
                             data,
                         }))
                         : undefined,
-                    cache_prompt: this.settings.cachePrompt,
                     temperature: this.settings.temperature,
                     top_k: this.settings.topK,
                     top_p: this.settings.topP,
+                    min_p: this.settings.minP,
                     n_predict: this.settings.maxGenerationTokens,
                     n_keep: this.settings.nKeep,
                     stop: this.settings.stopSequences,
@@ -68,12 +68,19 @@ class LlamaCppCompletionModel extends AbstractModel_js_1.AbstractModel {
                     repeat_penalty: this.settings.repeatPenalty,
                     repeat_last_n: this.settings.repeatLastN,
                     penalize_nl: this.settings.penalizeNl,
+                    presence_penalty: this.settings.presencePenalty,
+                    frequency_penalty: this.settings.frequencyPenalty,
+                    penalty_prompt: this.settings.penaltyPrompt,
                     mirostat: this.settings.mirostat,
                     mirostat_tau: this.settings.mirostatTau,
                     mirostat_eta: this.settings.mirostatEta,
+                    grammar: this.settings.grammar,
                     seed: this.settings.seed,
                     ignore_eos: this.settings.ignoreEos,
                     logit_bias: this.settings.logitBias,
+                    n_probs: this.settings.nProbs,
+                    cache_prompt: this.settings.cachePrompt,
+                    slot_id: this.settings.slotId,
                 },
                 failedResponseHandler: LlamaCppError_js_1.failedLlamaCppCallResponseHandler,
                 successfulResponseHandler: responseFormat.handler,
@@ -85,22 +92,29 @@ class LlamaCppCompletionModel extends AbstractModel_js_1.AbstractModel {
         const eventSettingProperties = [
             ...TextGenerationModel_js_1.textGenerationModelProperties,
             "contextWindowSize",
-            "cachePrompt",
             "temperature",
             "topK",
             "topP",
+            "minP",
             "nKeep",
             "tfsZ",
             "typicalP",
             "repeatPenalty",
             "repeatLastN",
             "penalizeNl",
+            "presencePenalty",
+            "frequencyPenalty",
+            "penaltyPrompt",
             "mirostat",
             "mirostatTau",
             "mirostatEta",
+            "grammar",
             "seed",
             "ignoreEos",
             "logitBias",
+            "nProbs",
+            "cachePrompt",
+            "slotId",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
@@ -221,7 +235,7 @@ const llamaCppTextGenerationResponseSchema = zod_1.z.object({
         predicted_n: zod_1.z.number(),
         predicted_per_second: zod_1.z.number().nullable(),
         predicted_per_token_ms: zod_1.z.number().nullable(),
-        prompt_ms: zod_1.z.number().nullable(),
+        prompt_ms: zod_1.z.number().nullable().optional(),
         prompt_n: zod_1.z.number(),
         prompt_per_second: zod_1.z.number().nullable(),
         prompt_per_token_ms: zod_1.z.number().nullable(),

package/model-provider/llamacpp/LlamaCppCompletionModel.d.ts

@@ -16,24 +16,111 @@ export interface LlamaCppCompletionModelSettings<CONTEXT_WINDOW_SIZE extends num
      */
     contextWindowSize?: CONTEXT_WINDOW_SIZE;
     /**
-     * Save the prompt and generation for avoid reprocess entire prompt if a part of this isn't change (default: false)
+     * Adjust the randomness of the generated text (default: 0.8).
      */
-    cachePrompt?: boolean;
     temperature?: number;
+    /**
+     * Limit the next token selection to the K most probable tokens (default: 40).
+     */
     topK?: number;
+    /**
+     * Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P (default: 0.95).
+     */
     topP?: number;
+    /**
+     * The minimum probability for a token to be considered, relative to the probability of the most likely token (default: 0.05).
+     */
+    minP?: number;
+    /**
+     * Specify the number of tokens from the prompt to retain when the context size is exceeded
+     * and tokens need to be discarded. By default, this value is set to 0 (meaning no tokens
+     * are kept). Use -1 to retain all tokens from the prompt.
+     */
     nKeep?: number;
+    /**
+     * Enable tail free sampling with parameter z (default: 1.0, 1.0 = disabled).
+     */
     tfsZ?: number;
+    /**
+     * Enable locally typical sampling with parameter p (default: 1.0, 1.0 = disabled).
+     */
     typicalP?: number;
+    /**
+     * Control the repetition of token sequences in the generated text (default: 1.1).
+     */
     repeatPenalty?: number;
+    /**
+     * Last n tokens to consider for penalizing repetition (default: 64, 0 = disabled, -1 = ctx-size).
+     */
     repeatLastN?: number;
+    /**
+     * Penalize newline tokens when applying the repeat penalty (default: true).
+     */
     penalizeNl?: boolean;
+    /**
+     * Repeat alpha presence penalty (default: 0.0, 0.0 = disabled).
+     */
+    presencePenalty?: number;
+    /**
+     * Repeat alpha frequency penalty (default: 0.0, 0.0 = disabled).
+     */
+    frequencyPenalty?: number;
+    /**
+     * This will replace the prompt for the purpose of the penalty evaluation.
+     * Can be either null, a string or an array of numbers representing tokens
+     * (default: null = use the original prompt).
+     */
+    penaltyPrompt?: string | number[];
+    /**
+     * Enable Mirostat sampling, controlling perplexity during text generation
+     * (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0).
+     */
     mirostat?: number;
+    /**
+     * Set the Mirostat target entropy, parameter tau (default: 5.0).
+     */
     mirostatTau?: number;
+    /**
+     * Set the Mirostat learning rate, parameter eta (default: 0.1).
+     */
     mirostatEta?: number;
+    /**
+     * Set grammar for grammar-based sampling (default: no grammar)
+     *
+     * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md
+     */
+    grammar?: string;
+    /**
+     * Set the random number generator (RNG) seed
+     * (default: -1, -1 = random seed).
+     */
     seed?: number;
+    /**
+     * Ignore end of stream token and continue generating (default: false).
+     */
     ignoreEos?: boolean;
+    /**
+     * Modify the likelihood of a token appearing in the generated text completion.
+     * For example, use "logit_bias": [[15043,1.0]] to increase the likelihood of the token
+     * 'Hello', or "logit_bias": [[15043,-1.0]] to decrease its likelihood.
+     * Setting the value to false, "logit_bias": [[15043,false]] ensures that the token Hello is
+     * never produced (default: []).
+     */
     logitBias?: Array<[number, number | false]>;
+    /**
+     * If greater than 0, the response also contains the probabilities of top N tokens
+     * for each generated token (default: 0)
+     */
+    nProbs?: number;
+    /**
+     * Save the prompt and generation for avoid reprocess entire prompt if a part of this isn't change (default: false)
+     */
+    cachePrompt?: boolean;
+    /**
+     * Assign the completion task to an specific slot.
+     * If is -1 the task will be assigned to a Idle slot (default: -1)
+     */
+    slotId?: number;
 }
 export interface LlamaCppCompletionPrompt {
     /**
@@ -96,10 +183,10 @@ export declare class LlamaCppCompletionModel<CONTEXT_WINDOW_SIZE extends number
                 predicted_n: number;
                 predicted_per_second: number | null;
                 predicted_per_token_ms: number | null;
-                prompt_ms: number | null;
                 prompt_n: number;
                 prompt_per_second: number | null;
                 prompt_per_token_ms: number | null;
+                prompt_ms?: number | null | undefined;
             };
             tokens_cached: number;
             tokens_evaluated: number;
@@ -155,10 +242,10 @@ export declare class LlamaCppCompletionModel<CONTEXT_WINDOW_SIZE extends number
             predicted_n: number;
             predicted_per_second: number | null;
             predicted_per_token_ms: number | null;
-            prompt_ms: number | null;
             prompt_n: number;
             prompt_per_second: number | null;
             prompt_per_token_ms: number | null;
+            prompt_ms?: number | null | undefined;
         };
         tokens_cached: number;
         tokens_evaluated: number;
@@ -267,7 +354,7 @@ declare const llamaCppTextGenerationResponseSchema: z.ZodObject<{
         predicted_n: z.ZodNumber;
         predicted_per_second: z.ZodNullable<z.ZodNumber>;
         predicted_per_token_ms: z.ZodNullable<z.ZodNumber>;
-        prompt_ms: z.ZodNullable<z.ZodNumber>;
+        prompt_ms: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
         prompt_n: z.ZodNumber;
         prompt_per_second: z.ZodNullable<z.ZodNumber>;
         prompt_per_token_ms: z.ZodNullable<z.ZodNumber>;
@@ -276,19 +363,19 @@ declare const llamaCppTextGenerationResponseSchema: z.ZodObject<{
         predicted_n: number;
         predicted_per_second: number | null;
         predicted_per_token_ms: number | null;
-        prompt_ms: number | null;
         prompt_n: number;
         prompt_per_second: number | null;
         prompt_per_token_ms: number | null;
+        prompt_ms?: number | null | undefined;
     }, {
         predicted_ms: number;
         predicted_n: number;
         predicted_per_second: number | null;
         predicted_per_token_ms: number | null;
-        prompt_ms: number | null;
         prompt_n: number;
         prompt_per_second: number | null;
         prompt_per_token_ms: number | null;
+        prompt_ms?: number | null | undefined;
     }>;
     tokens_cached: z.ZodNumber;
     tokens_evaluated: z.ZodNumber;
@@ -333,10 +420,10 @@ declare const llamaCppTextGenerationResponseSchema: z.ZodObject<{
         predicted_n: number;
         predicted_per_second: number | null;
         predicted_per_token_ms: number | null;
-        prompt_ms: number | null;
         prompt_n: number;
         prompt_per_second: number | null;
         prompt_per_token_ms: number | null;
+        prompt_ms?: number | null | undefined;
     };
     tokens_cached: number;
     tokens_evaluated: number;
@@ -381,10 +468,10 @@ declare const llamaCppTextGenerationResponseSchema: z.ZodObject<{
         predicted_n: number;
         predicted_per_second: number | null;
         predicted_per_token_ms: number | null;
-        prompt_ms: number | null;
         prompt_n: number;
         prompt_per_second: number | null;
         prompt_per_token_ms: number | null;
+        prompt_ms?: number | null | undefined;
     };
     tokens_cached: number;
     tokens_evaluated: number;
@@ -431,10 +518,10 @@ declare const llamaCppTextStreamChunkSchema: import("../../core/schema/ZodSchema
         predicted_n: number;
         predicted_per_second: number | null;
         predicted_per_token_ms: number | null;
-        prompt_ms: number | null;
         prompt_n: number;
         prompt_per_second: number | null;
         prompt_per_token_ms: number | null;
+        prompt_ms?: number | null | undefined;
     };
     tokens_cached: number;
     tokens_evaluated: number;
@@ -494,10 +581,10 @@ export declare const LlamaCppCompletionResponseFormat: {
                 predicted_n: number;
                 predicted_per_second: number | null;
                 predicted_per_token_ms: number | null;
-                prompt_ms: number | null;
                 prompt_n: number;
                 prompt_per_second: number | null;
                 prompt_per_token_ms: number | null;
+                prompt_ms?: number | null | undefined;
             };
             tokens_cached: number;
             tokens_evaluated: number;
@@ -552,10 +639,10 @@ export declare const LlamaCppCompletionResponseFormat: {
                 predicted_n: number;
                 predicted_per_second: number | null;
                 predicted_per_token_ms: number | null;
-                prompt_ms: number | null;
                 prompt_n: number;
                 prompt_per_second: number | null;
                 prompt_per_token_ms: number | null;
+                prompt_ms?: number | null | undefined;
             };
             tokens_cached: number;
             tokens_evaluated: number;

package/model-provider/llamacpp/LlamaCppCompletionModel.js

@@ -53,10 +53,10 @@ export class LlamaCppCompletionModel extends AbstractModel {
                             data,
                         }))
                         : undefined,
-                    cache_prompt: this.settings.cachePrompt,
                     temperature: this.settings.temperature,
                     top_k: this.settings.topK,
                     top_p: this.settings.topP,
+                    min_p: this.settings.minP,
                     n_predict: this.settings.maxGenerationTokens,
                     n_keep: this.settings.nKeep,
                     stop: this.settings.stopSequences,
@@ -65,12 +65,19 @@ export class LlamaCppCompletionModel extends AbstractModel {
                     repeat_penalty: this.settings.repeatPenalty,
                     repeat_last_n: this.settings.repeatLastN,
                     penalize_nl: this.settings.penalizeNl,
+                    presence_penalty: this.settings.presencePenalty,
+                    frequency_penalty: this.settings.frequencyPenalty,
+                    penalty_prompt: this.settings.penaltyPrompt,
                     mirostat: this.settings.mirostat,
                     mirostat_tau: this.settings.mirostatTau,
                     mirostat_eta: this.settings.mirostatEta,
+                    grammar: this.settings.grammar,
                     seed: this.settings.seed,
                     ignore_eos: this.settings.ignoreEos,
                     logit_bias: this.settings.logitBias,
+                    n_probs: this.settings.nProbs,
+                    cache_prompt: this.settings.cachePrompt,
+                    slot_id: this.settings.slotId,
                 },
                 failedResponseHandler: failedLlamaCppCallResponseHandler,
                 successfulResponseHandler: responseFormat.handler,
@@ -82,22 +89,29 @@ export class LlamaCppCompletionModel extends AbstractModel {
         const eventSettingProperties = [
             ...textGenerationModelProperties,
             "contextWindowSize",
-            "cachePrompt",
             "temperature",
             "topK",
             "topP",
+            "minP",
             "nKeep",
             "tfsZ",
             "typicalP",
             "repeatPenalty",
             "repeatLastN",
             "penalizeNl",
+            "presencePenalty",
+            "frequencyPenalty",
+            "penaltyPrompt",
             "mirostat",
             "mirostatTau",
             "mirostatEta",
+            "grammar",
             "seed",
             "ignoreEos",
             "logitBias",
+            "nProbs",
+            "cachePrompt",
+            "slotId",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
@@ -217,7 +231,7 @@ const llamaCppTextGenerationResponseSchema = z.object({
         predicted_n: z.number(),
         predicted_per_second: z.number().nullable(),
         predicted_per_token_ms: z.number().nullable(),
-        prompt_ms: z.number().nullable(),
+        prompt_ms: z.number().nullable().optional(),
         prompt_n: z.number(),
         prompt_per_second: z.number().nullable(),
         prompt_per_token_ms: z.number().nullable(),

package/model-provider/openai/AbstractOpenAIChatModel.cjs

@@ -232,7 +232,7 @@ const openAIChatResponseSchema = zod_1.z.object({
         total_tokens: zod_1.z.number(),
     }),
 });
-const chatCompletionChunkSchema = zod_1.z.object({
+const openaiChatChunkSchema = (0, ZodSchema_js_1.zodSchema)(zod_1.z.object({
     object: zod_1.z.literal("chat.completion.chunk"),
     id: zod_1.z.string(),
     choices: zod_1.z.array(zod_1.z.object({
@@ -271,15 +271,7 @@ const chatCompletionChunkSchema = zod_1.z.object({
     created: zod_1.z.number(),
     model: zod_1.z.string(),
     system_fingerprint: zod_1.z.string().optional().nullable(),
-});
-const openaiChatChunkSchema = (0, ZodSchema_js_1.zodSchema)(zod_1.z.union([
-    chatCompletionChunkSchema,
-    zod_1.z.object({
-        object: zod_1.z.string().refine((obj) => obj !== "chat.completion.chunk", {
-            message: "Object must be 'chat.completion.chunk'",
-        }),
-    }),
-]));
+}));
 exports.OpenAIChatResponseFormat = {
     /**
      * Returns the response as a JSON object.