modelfusion 0.116.1 → 0.117.0

package/CHANGELOG.md

@@ -1,5 +1,46 @@
 # Changelog
 
+## v0.117.0 - 2024-01-06
+
+### Added
+
+- Predefined Llama.cpp GBNF grammars:
+
+  - `llamacpp.grammar.json`: Restricts the output to JSON.
+  - `llamacpp.grammar.jsonArray`: Restricts the output to a JSON array.
+  - `llamacpp.grammar.list`: Restricts the output to a newline-separated list where each line starts with `- `.
+
+- Llama.cpp structure generation support:
+
+  ```ts
+  const structure = await generateStructure(
+    llamacpp
+      .TextGenerator({
+        // run openhermes-2.5-mistral-7b.Q4_K_M.gguf in llama.cpp
+        maxGenerationTokens: 1024,
+        temperature: 0,
+      })
+      .withTextPromptTemplate(ChatMLPrompt.instruction()) // needed for jsonStructurePrompt.text()
+      .asStructureGenerationModel(jsonStructurePrompt.text()), // automatically restrict the output to JSON
+
+    zodSchema(
+      z.object({
+        characters: z.array(
+          z.object({
+            name: z.string(),
+            class: z
+              .string()
+              .describe("Character class, e.g. warrior, mage, or thief."),
+            description: z.string(),
+          })
+        ),
+      })
+    ),
+
+    "Generate 3 character descriptions for a fantasy role playing game. "
+  );
+  ```
+
 ## v0.116.0 - 2024-01-05
 
 ### Added

package/README.md

@@ -152,7 +152,7 @@ const sentiment = await generateStructure(
 );
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Ollama](https://modelfusion.dev//integration/model-provider/ollama)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Ollama](https://modelfusion.dev//integration/model-provider/ollama), [Llama.cpp](https://modelfusion.dev//integration/model-provider/llama.cpp)
 
 #### streamStructure
 
@@ -198,7 +198,7 @@ for await (const part of structureStream) {
 }
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Ollama](https://modelfusion.dev//integration/model-provider/ollama)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Ollama](https://modelfusion.dev//integration/model-provider/ollama), [Llama.cpp](https://modelfusion.dev//integration/model-provider/llama.cpp)
 
 ### [Generate Image](https://modelfusion.dev/guide/function/generate-image)
 
@@ -563,6 +563,8 @@ modelfusion.setLogFormat("detailed-object"); // log full events
   - [Split Text](https://modelfusion.dev/guide/text-chunk/split)
 - [Utilities](https://modelfusion.dev/guide/util/)
   - [API Configuration](https://modelfusion.dev/guide/util/api-configuration)
+    - [Base URL](https://modelfusion.dev/guide/util/api-configuration/base-url)
+    - [Headers](https://modelfusion.dev/guide/util/api-configuration/headers)
     - [Retry strategies](https://modelfusion.dev/guide/util/api-configuration/retry)
     - [Throttling strategies](https://modelfusion.dev/guide/util/api-configuration/throttle)
   - [Logging](https://modelfusion.dev/guide/util/logging)

package/model-provider/llamacpp/LlamaCppCompletionModel.cjs

@@ -15,6 +15,7 @@ const parseEventSourceStream_js_1 = require("../../util/streaming/parseEventSour
 const LlamaCppApiConfiguration_js_1 = require("./LlamaCppApiConfiguration.cjs");
 const LlamaCppError_js_1 = require("./LlamaCppError.cjs");
 const LlamaCppTokenizer_js_1 = require("./LlamaCppTokenizer.cjs");
+const LlamaCppGrammars_js_1 = require("./LlamaCppGrammars.cjs");
 class LlamaCppCompletionModel extends AbstractModel_js_1.AbstractModel {
     constructor(settings = {}) {
         super({ settings });
@@ -167,6 +168,12 @@ class LlamaCppCompletionModel extends AbstractModel_js_1.AbstractModel {
     extractTextDelta(delta) {
         return delta.content;
     }
+    withJsonOutput() {
+        // don't override the grammar if it's already set (to support more restrictive grammars)
+        return this.settings.grammar == null
+            ? this.withSettings({ grammar: LlamaCppGrammars_js_1.json })
+            : this;
+    }
     withTextPrompt() {
         return this.withPromptTemplate({
             format(prompt) {

package/model-provider/llamacpp/LlamaCppCompletionModel.d.ts

@@ -376,6 +376,7 @@ export declare class LlamaCppCompletionModel<CONTEXT_WINDOW_SIZE extends number
         content: string;
     }>>>;
     extractTextDelta(delta: unknown): string;
+    withJsonOutput(): this;
     withTextPrompt(): PromptTemplateTextStreamingModel<string, LlamaCppCompletionPrompt, LlamaCppCompletionModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     /**
      * Maps the prompt for a text version of the Llama.cpp prompt template (without image support).

package/model-provider/llamacpp/LlamaCppCompletionModel.js

@@ -12,6 +12,7 @@ import { parseEventSourceStream } from "../../util/streaming/parseEventSourceStr
 import { LlamaCppApiConfiguration } from "./LlamaCppApiConfiguration.js";
 import { failedLlamaCppCallResponseHandler } from "./LlamaCppError.js";
 import { LlamaCppTokenizer } from "./LlamaCppTokenizer.js";
+import { json } from "./LlamaCppGrammars.js";
 export class LlamaCppCompletionModel extends AbstractModel {
     constructor(settings = {}) {
         super({ settings });
@@ -164,6 +165,12 @@ export class LlamaCppCompletionModel extends AbstractModel {
     extractTextDelta(delta) {
         return delta.content;
     }
+    withJsonOutput() {
+        // don't override the grammar if it's already set (to support more restrictive grammars)
+        return this.settings.grammar == null
+            ? this.withSettings({ grammar: json })
+            : this;
+    }
     withTextPrompt() {
         return this.withPromptTemplate({
             format(prompt) {

package/model-provider/llamacpp/LlamaCppFacade.cjs

@@ -1,6 +1,29 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Tokenizer = exports.TextEmbedder = exports.TextGenerator = exports.Api = void 0;
+exports.grammar = exports.Tokenizer = exports.TextEmbedder = exports.TextGenerator = exports.Api = void 0;
 const LlamaCppApiConfiguration_js_1 = require("./LlamaCppApiConfiguration.cjs");
 const LlamaCppCompletionModel_js_1 = require("./LlamaCppCompletionModel.cjs");
 const LlamaCppTextEmbeddingModel_js_1 = require("./LlamaCppTextEmbeddingModel.cjs");
@@ -25,3 +48,7 @@ function Tokenizer(api = new LlamaCppApiConfiguration_js_1.LlamaCppApiConfigurat
     return new LlamaCppTokenizer_js_1.LlamaCppTokenizer(api);
 }
 exports.Tokenizer = Tokenizer;
+/**
+ * GBNF grammars. You can use them in the `grammar` option of the `TextGenerator` model.
+ */
+exports.grammar = __importStar(require("./LlamaCppGrammars.cjs"));

package/model-provider/llamacpp/LlamaCppFacade.d.ts

@@ -12,3 +12,7 @@ export declare function Api(settings: PartialBaseUrlPartsApiConfigurationOptions
 export declare function TextGenerator<CONTEXT_WINDOW_SIZE extends number>(settings?: LlamaCppCompletionModelSettings<CONTEXT_WINDOW_SIZE>): LlamaCppCompletionModel<CONTEXT_WINDOW_SIZE>;
 export declare function TextEmbedder(settings?: LlamaCppTextEmbeddingModelSettings): LlamaCppTextEmbeddingModel;
 export declare function Tokenizer(api?: ApiConfiguration): LlamaCppTokenizer;
+/**
+ * GBNF grammars. You can use them in the `grammar` option of the `TextGenerator` model.
+ */
+export * as grammar from "./LlamaCppGrammars.js";

package/model-provider/llamacpp/LlamaCppFacade.js

@@ -18,3 +18,7 @@ export function TextEmbedder(settings = {}) {
 export function Tokenizer(api = new LlamaCppApiConfiguration()) {
     return new LlamaCppTokenizer(api);
 }
+/**
+ * GBNF grammars. You can use them in the `grammar` option of the `TextGenerator` model.
+ */
+export * as grammar from "./LlamaCppGrammars.js";

package/model-provider/llamacpp/LlamaCppGrammars.cjs

@@ -0,0 +1,84 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.list = exports.jsonArray = exports.json = void 0;
+/**
+ * GBNF grammar for JSON.
+ *
+ * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/json.gbnf
+ */
+exports.json = `
+root   ::= object
+value  ::= object | array | string | number | ("true" | "false" | "null") ws
+
+object ::=
+  "{" ws (
+            string ":" ws value
+    ("," ws string ":" ws value)*
+  )? "}" ws
+
+array  ::=
+  "[" ws (
+            value
+    ("," ws value)*
+  )? "]" ws
+
+string ::=
+  "\\"" (
+    [^"\\\\] |
+    "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+  )* "\\"" ws
+
+number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= ([ \t\n] ws)?
+`;
+/**
+ * GBNF grammar for JSON array outputs. Restricts whitespace at the end of the array.
+ *
+ * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/json_arr.gbnf
+ */
+exports.jsonArray = `
+root   ::= arr
+value  ::= object | array | string | number | ("true" | "false" | "null") ws
+
+arr  ::=
+  "[\n" ws (
+            value
+    (",\n" ws value)*
+  )? "]"
+
+object ::=
+  "{" ws (
+            string ":" ws value
+    ("," ws string ":" ws value)*
+  )? "}" ws
+
+array  ::=
+  "[" ws (
+            value
+    ("," ws value)*
+  )? "]" ws
+
+string ::=
+  "\\"" (
+    [^"\\\\] |
+    "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+  )* "\\"" ws
+
+number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= ([ \t\n] ws)?
+`;
+/**
+ * GBNF grammar for list outputs. List items are separated by newlines and start with `- `.
+ *
+ * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/list.gbnf
+ */
+exports.list = `
+root ::= item+
+
+# Excludes various line break characters
+item ::= "- " [^\r\n\x0b\x0c\x85\u2028\u2029]+ "\n"
+`;

package/model-provider/llamacpp/LlamaCppGrammars.d.ts

@@ -0,0 +1,18 @@
+/**
+ * GBNF grammar for JSON.
+ *
+ * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/json.gbnf
+ */
+export declare const json: string;
+/**
+ * GBNF grammar for JSON array outputs. Restricts whitespace at the end of the array.
+ *
+ * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/json_arr.gbnf
+ */
+export declare const jsonArray: string;
+/**
+ * GBNF grammar for list outputs. List items are separated by newlines and start with `- `.
+ *
+ * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/list.gbnf
+ */
+export declare const list: string;

package/model-provider/llamacpp/LlamaCppGrammars.js

@@ -0,0 +1,81 @@
+/**
+ * GBNF grammar for JSON.
+ *
+ * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/json.gbnf
+ */
+export const json = `
+root   ::= object
+value  ::= object | array | string | number | ("true" | "false" | "null") ws
+
+object ::=
+  "{" ws (
+            string ":" ws value
+    ("," ws string ":" ws value)*
+  )? "}" ws
+
+array  ::=
+  "[" ws (
+            value
+    ("," ws value)*
+  )? "]" ws
+
+string ::=
+  "\\"" (
+    [^"\\\\] |
+    "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+  )* "\\"" ws
+
+number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= ([ \t\n] ws)?
+`;
+/**
+ * GBNF grammar for JSON array outputs. Restricts whitespace at the end of the array.
+ *
+ * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/json_arr.gbnf
+ */
+export const jsonArray = `
+root   ::= arr
+value  ::= object | array | string | number | ("true" | "false" | "null") ws
+
+arr  ::=
+  "[\n" ws (
+            value
+    (",\n" ws value)*
+  )? "]"
+
+object ::=
+  "{" ws (
+            string ":" ws value
+    ("," ws string ":" ws value)*
+  )? "}" ws
+
+array  ::=
+  "[" ws (
+            value
+    ("," ws value)*
+  )? "]" ws
+
+string ::=
+  "\\"" (
+    [^"\\\\] |
+    "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+  )* "\\"" ws
+
+number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= ([ \t\n] ws)?
+`;
+/**
+ * GBNF grammar for list outputs. List items are separated by newlines and start with `- `.
+ *
+ * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/list.gbnf
+ */
+export const list = `
+root ::= item+
+
+# Excludes various line break characters
+item ::= "- " [^\r\n\x0b\x0c\x85\u2028\u2029]+ "\n"
+`;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "The TypeScript library for building AI applications.",
-  "version": "0.116.1",
+  "version": "0.117.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [