node-llama-cpp 3.0.0-beta.11 → 3.0.0-beta.13

package/README.md

@@ -1,5 +1,5 @@
 <div align="center">
-    <img alt="node-llama-cpp Logo" src="https://media.githubusercontent.com/media/withcatai/node-llama-cpp/master/assets/logo.roundEdges.png" width="360px" />
+    <img alt="node-llama-cpp Logo" src="https://raw.githubusercontent.com/withcatai/node-llama-cpp/master/assets/logo.roundEdges.png" width="360px" />
     <h1>node-llama-cpp</h1>
     <p>Run AI models locally on your machine</p>
     <sub>Pre-built bindings are provided with a fallback to building from source with cmake</sub>
@@ -15,11 +15,11 @@
 
 </div>
 
-✨ New! [Try the beta of version `3.0.0`](https://github.com/withcatai/node-llama-cpp/pull/105) ✨ (included: function calling, automatic chat wrapper detection, and more)
+✨ New! [Try the beta of version `3.0.0`](https://github.com/withcatai/node-llama-cpp/pull/105) ✨ (included: function calling, automatic chat wrapper detection, embedding support, and more)
 
 ## Features
 * Run a text generation model locally on your machine
-* Metal and CUDA support
+* Metal, CUDA and Vulkan support
 * Pre-built binaries are provided, with a fallback to building from source without `node-gyp` or Python
 * Chat with a model using a chat wrapper
 * Use the CLI to chat with a model without writing any code
@@ -84,7 +84,7 @@ To contribute to `node-llama-cpp` read the [contribution guide](https://withcata
 <br />
 
 <div align="center" width="360">
-    <img alt="Star please" src="https://media.githubusercontent.com/media/withcatai/node-llama-cpp/master/assets/star.please.roundEdges.png" width="360" margin="auto" />
+    <img alt="Star please" src="https://raw.githubusercontent.com/withcatai/node-llama-cpp/master/assets/star.please.roundEdges.png" width="360" margin="auto" />
     <br/>
     <p align="right">
         <i>If you like this repo, star it ✨</i>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;

package/dist/ChatWrapper.d.ts

@@ -15,6 +15,7 @@ export type ChatWrapperSettings = {
     };
 };
 export declare abstract class ChatWrapper {
+    static defaultSetting: ChatWrapperSettings;
     abstract readonly wrapperName: string;
     readonly settings: ChatWrapperSettings;
     generateContextText(history: readonly ChatHistoryItem[], { availableFunctions, documentFunctionParams }?: {

package/dist/ChatWrapper.js

@@ -1,7 +1,7 @@
 import { LlamaText } from "./utils/LlamaText.js";
 import { getTypeScriptTypeStringForGbnfJsonSchema } from "./utils/getTypeScriptTypeStringForGbnfJsonSchema.js";
 export class ChatWrapper {
-    settings = {
+    static defaultSetting = {
         functions: {
             call: {
                 optionalPrefixSpace: true,
@@ -15,6 +15,7 @@ export class ChatWrapper {
             }
         }
     };
+    settings = ChatWrapper.defaultSetting;
     generateContextText(history, { availableFunctions, documentFunctionParams } = {}) {
         const historyWithFunctions = this.addAvailableFunctionsSystemMessageToHistory(history, availableFunctions, {
             documentParams: documentFunctionParams

package/dist/ChatWrapper.js.map

@@ -1 +1 @@
-{"version":3,"file":"ChatWrapper.js","sourceRoot":"","sources":["../src/ChatWrapper.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAC,wCAAwC,EAAC,MAAM,qDAAqD,CAAC;AAiB7G,MAAM,OAAgB,WAAW;IAGb,QAAQ,GAAwB;QAC5C,SAAS,EAAE;YACP,IAAI,EAAE;gBACF,mBAAmB,EAAE,IAAI;gBACzB,MAAM,EAAE,UAAU;gBAClB,YAAY,EAAE,GAAG;gBACjB,MAAM,EAAE,KAAK;aAChB;YACD,MAAM,EAAE;gBACJ,MAAM,EAAE,aAAa;gBACrB,MAAM,EAAE,IAAI;aACf;SACJ;KACJ,CAAC;IAEK,mBAAmB,CAAC,OAAmC,EAAE,EAAC,kBAAkB,EAAE,sBAAsB,KAGvG,EAAE;QASF,MAAM,oBAAoB,GAAG,IAAI,CAAC,2CAA2C,CAAC,OAAO,EAAE,kBAAkB,EAAE;YACvG,cAAc,EAAE,sBAAsB;SACzC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,oBAAoB;aAC7B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;gBACtB,OAAO,SAAS,CAAC,WAAW,IAAI,CAAC,IAAI,EAAE,CAAC;iBACvC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;gBACzB,OAAO,SAAS,CAAC,SAAS,IAAI,CAAC,IAAI,EAAE,CAAC;iBACrC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;gBAC1B,OAAO,SAAS,CAAC,UAAU,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAE/E,OAAO,IAAoB,CAAC;QAChC,CAAC,CAAC,CAAC;QAEP,OAAO;YACH,WAAW,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC;YAC9C,sBAAsB,EAAE,EAAE;SAC7B,CAAC;IACN,CAAC;IAEM,6BAA6B,CAAC,IAAY,EAAE,MAAW,EAAE,MAAW;QACvE,OAAO,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3G,CAAC;IAEM,oBAAoB,CAAC,IAAY,EAAE,MAAW;QACjD,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM;YACtC,IAAI;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY;YACzC,CACI,MAAM,KAAK,SAAS;gBAChB,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAC/B;YACD,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5C,CAAC;IAEM,0BAA0B,CAAC,YAAoB,EAAE,cAAmB,EAAE,MAAW;QACpF,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAE,EAAE,CACvC,IAAI,CAAC,UAAU,CAAC,kBAAkB,EAAE,YAAY,CAAC;aAC5C,UAAU,CAAC,oBAAoB,EAAE,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;QAE9G,OAAO,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;YAC3D,CACI,MAAM,KAAK,SAAS;gBAChB,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAC/B;YACD,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjE,CAAC;IAEM,yBAAyB,CAAC,aAA4C;QACzE,OAAO,aAAa;aACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,IAAI,OAAO,IAAI,KAAK,QAAQ;gBACxB,OAAO,IAAI,CAAC;YAEhB,OAAO,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/F,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAEM,oCAAoC,CAAC,kBAAsC,EAAE,EAAC,cAAc,GAAG,IAAI,EAEzG;QACG,MAAM,sBAAsB,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;QAErE,IAAI,sBAAsB,CAAC,MAAM,KAAK,CAAC;YACnC,OAAO,EAAE,CAAC;QAEd,OAAO,+HAA+H;YAClI,4BAA4B;YAC5B,sBAAsB;iBACjB,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE;gBAClB,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;gBAC5D,IAAI,GAAG,GAAG,EAAE,CAAC;gBAEb,IAAI,kBAAkB,EAAE,WAAW,IAAI,IAAI,IAAI,kBAAkB,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE;oBACvF,GAAG,IAAI,KAAK,GAAG,kBAAkB,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;gBAEnF,GAAG,IAAI,WAAW,GAAG,YAAY,GAAG,GAAG,CAAC;gBAExC,IAAI,cAAc,IAAI,kBAAkB,EAAE,MAAM,IAAI,IAAI;oBACpD,GAAG,IAAI,UAAU,GAAG,wCAAwC,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;qBACvF,IAAI,CAAC,cAAc,IAAI,kBAAkB,EAAE,MAAM,IAAI,IAAI;oBAC1D,GAAG,IAAI,QAAQ,CAAC;gBAEpB,GAAG,IAAI,IAAI,CAAC;gBAEZ,OAAO,GAAG,CAAC;YACf,CAAC,CAAC;iBACD,IAAI,CAAC,MAAM,CAAC;YACjB,WAAW;YAEX,gEAAgE;YAChE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;YAC/C,cAAc;YACd,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY;YACzC,0BAA0B;YAC1B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM;YAE5C,yLAAyL;YACzL,uFAAuF,CAAC;IAChG,CAAC;IAEM,2CAA2C,CAAC,OAAmC,EAAE,kBAAuC,EAAE,EAC7H,cAAc,GAAG,IAAI,KAGrB,EAAE;QACF,MAAM,sBAAsB,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;QAErE,IAAI,kBAAkB,IAAI,IAAI,IAAI,sBAAsB,CAAC,MAAM,KAAK,CAAC;YACjE,OAAO,OAAO,CAAC;QAEnB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAE5B,MAAM,0BAA0B,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACnF,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,0BAA0B,CAAC,EAAE,CAAC,EAAE;YACnD,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,IAAI,CAAC,oCAAoC,CAAC,kBAAkB,EAAE,EAAC,cAAc,EAAC,CAAC;SACxF,CAAC,CAAC;QAEH,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;;OAGG;IACI,2BAA2B,CAAC,EAAC,0BAA0B,EAAwC;QAClG,IAAI,0BAA0B;YAC1B,OAAO,EAAE,CAAC;QAEd,OAAO,EAAE,CAAC;IACd,CAAC;CACJ"}
+{"version":3,"file":"ChatWrapper.js","sourceRoot":"","sources":["../src/ChatWrapper.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAC,wCAAwC,EAAC,MAAM,qDAAqD,CAAC;AAiB7G,MAAM,OAAgB,WAAW;IACtB,MAAM,CAAC,cAAc,GAAwB;QAChD,SAAS,EAAE;YACP,IAAI,EAAE;gBACF,mBAAmB,EAAE,IAAI;gBACzB,MAAM,EAAE,UAAU;gBAClB,YAAY,EAAE,GAAG;gBACjB,MAAM,EAAE,KAAK;aAChB;YACD,MAAM,EAAE;gBACJ,MAAM,EAAE,aAAa;gBACrB,MAAM,EAAE,IAAI;aACf;SACJ;KACJ,CAAC;IAGc,QAAQ,GAAwB,WAAW,CAAC,cAAc,CAAC;IAEpE,mBAAmB,CAAC,OAAmC,EAAE,EAAC,kBAAkB,EAAE,sBAAsB,KAGvG,EAAE;QASF,MAAM,oBAAoB,GAAG,IAAI,CAAC,2CAA2C,CAAC,OAAO,EAAE,kBAAkB,EAAE;YACvG,cAAc,EAAE,sBAAsB;SACzC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,oBAAoB;aAC7B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;gBACtB,OAAO,SAAS,CAAC,WAAW,IAAI,CAAC,IAAI,EAAE,CAAC;iBACvC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;gBACzB,OAAO,SAAS,CAAC,SAAS,IAAI,CAAC,IAAI,EAAE,CAAC;iBACrC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;gBAC1B,OAAO,SAAS,CAAC,UAAU,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAE/E,OAAO,IAAoB,CAAC;QAChC,CAAC,CAAC,CAAC;QAEP,OAAO;YACH,WAAW,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC;YAC9C,sBAAsB,EAAE,EAAE;SAC7B,CAAC;IACN,CAAC;IAEM,6BAA6B,CAAC,IAAY,EAAE,MAAW,EAAE,MAAW;QACvE,OAAO,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3G,CAAC;IAEM,oBAAoB,CAAC,IAAY,EAAE,MAAW;QACjD,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM;YACtC,IAAI;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY;YACzC,CACI,MAAM,KAAK,SAAS;gBAChB,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAC/B;YACD,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5C,CAAC;IAEM,0BAA0B,CAAC,YAAoB,EAAE,cAAmB,EAAE,MAAW;QACpF,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAE,EAAE,CACvC,IAAI,CAAC,UAAU,CAAC,kBAAkB,EAAE,YAAY,CAAC;aAC5C,UAAU,CAAC,oBAAoB,EAAE,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;QAE9G,OAAO,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;YAC3D,CACI,MAAM,KAAK,SAAS;gBAChB,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAC/B;YACD,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjE,CAAC;IAEM,yBAAyB,CAAC,aAA4C;QACzE,OAAO,aAAa;aACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,IAAI,OAAO,IAAI,KAAK,QAAQ;gBACxB,OAAO,IAAI,CAAC;YAEhB,OAAO,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/F,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAEM,oCAAoC,CAAC,kBAAsC,EAAE,EAAC,cAAc,GAAG,IAAI,EAEzG;QACG,MAAM,sBAAsB,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;QAErE,IAAI,sBAAsB,CAAC,MAAM,KAAK,CAAC;YACnC,OAAO,EAAE,CAAC;QAEd,OAAO,+HAA+H;YAClI,4BAA4B;YAC5B,sBAAsB;iBACjB,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE;gBAClB,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;gBAC5D,IAAI,GAAG,GAAG,EAAE,CAAC;gBAEb,IAAI,kBAAkB,EAAE,WAAW,IAAI,IAAI,IAAI,kBAAkB,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE;oBACvF,GAAG,IAAI,KAAK,GAAG,kBAAkB,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;gBAEnF,GAAG,IAAI,WAAW,GAAG,YAAY,GAAG,GAAG,CAAC;gBAExC,IAAI,cAAc,IAAI,kBAAkB,EAAE,MAAM,IAAI,IAAI;oBACpD,GAAG,IAAI,UAAU,GAAG,wCAAwC,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;qBACvF,IAAI,CAAC,cAAc,IAAI,kBAAkB,EAAE,MAAM,IAAI,IAAI;oBAC1D,GAAG,IAAI,QAAQ,CAAC;gBAEpB,GAAG,IAAI,IAAI,CAAC;gBAEZ,OAAO,GAAG,CAAC;YACf,CAAC,CAAC;iBACD,IAAI,CAAC,MAAM,CAAC;YACjB,WAAW;YAEX,gEAAgE;YAChE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;YAC/C,cAAc;YACd,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY;YACzC,0BAA0B;YAC1B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM;YAE5C,yLAAyL;YACzL,uFAAuF,CAAC;IAChG,CAAC;IAEM,2CAA2C,CAAC,OAAmC,EAAE,kBAAuC,EAAE,EAC7H,cAAc,GAAG,IAAI,KAGrB,EAAE;QACF,MAAM,sBAAsB,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;QAErE,IAAI,kBAAkB,IAAI,IAAI,IAAI,sBAAsB,CAAC,MAAM,KAAK,CAAC;YACjE,OAAO,OAAO,CAAC;QAEnB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAE5B,MAAM,0BAA0B,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACnF,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,0BAA0B,CAAC,EAAE,CAAC,EAAE;YACnD,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,IAAI,CAAC,oCAAoC,CAAC,kBAAkB,EAAE,EAAC,cAAc,EAAC,CAAC;SACxF,CAAC,CAAC;QAEH,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;;OAGG;IACI,2BAA2B,CAAC,EAAC,0BAA0B,EAAwC;QAClG,IAAI,0BAA0B;YAC1B,OAAO,EAAE,CAAC;QAEd,OAAO,EAAE,CAAC;IACd,CAAC"}

package/dist/TemplateChatWrapper.d.ts

@@ -0,0 +1,67 @@
+import { ChatHistoryItem, ChatModelFunctions } from "./types.js";
+import { LlamaText } from "./utils/LlamaText.js";
+import { ChatWrapper, ChatWrapperSettings } from "./ChatWrapper.js";
+/**
+ * A chat wrapper based on a simple template.
+ *
+ * Example usage:
+ * ```typescript
+ * const chatWrapper = new TemplateChatWrapper({
+ *     template: "{{systemPrompt}}\n{{history}}model:{{completion}}\nuser:",
+ *     historyTemplate: "{{roleName}}: {{message}}\n",
+ *     modelRoleName: "model",
+ *     userRoleName: "user",
+ *     systemRoleName: "system", // optional
+ *     // functionCallMessageTemplate: [ // optional
+ *     //     "[[call: {{functionName}}({{functionParams}})]]",
+ *     //     " [[result: {{functionCallResult}}]]"
+ *     // ]
+ * });
+ * ```
+ *
+ * **`{{systemPrompt}}`** is optional and is replaced with the first system message
+ * (when is does, that system message is not included in the history).
+ *
+ * **`{{history}}`** is replaced with the chat history.
+ * Each message in the chat history is converted using template passed to `historyTemplate`, and all messages are joined together.
+ *
+ * **`{{completion}}`** is where the model's response is generated.
+ * The text that comes after `{{completion}}` is used to determine when the model has finished generating the response,
+ * and thus is mandatory.
+ *
+ * **`functionCallMessageTemplate`** is used to specify the format in which functions can be called by the model and
+ * how their results are fed to the model after the function call.
+ */
+export declare class TemplateChatWrapper extends ChatWrapper {
+    readonly wrapperName = "Template";
+    readonly settings: ChatWrapperSettings;
+    readonly template: ChatTemplate;
+    readonly historyTemplate: ChatHistoryTemplate;
+    readonly modelRoleName: string;
+    readonly userRoleName: string;
+    readonly systemRoleName: string;
+    readonly joinAdjacentMessagesOfTheSameType: boolean;
+    constructor({ template, historyTemplate, modelRoleName, userRoleName, systemRoleName, functionCallMessageTemplate, joinAdjacentMessagesOfTheSameType }: {
+        template: ChatTemplate;
+        historyTemplate: ChatHistoryTemplate;
+        modelRoleName: string;
+        userRoleName: string;
+        systemRoleName?: string;
+        functionCallMessageTemplate?: ChatHistoryFunctionCallMessageTemplate;
+        joinAdjacentMessagesOfTheSameType?: boolean;
+    });
+    generateContextText(history: readonly ChatHistoryItem[], { availableFunctions, documentFunctionParams }?: {
+        availableFunctions?: ChatModelFunctions;
+        documentFunctionParams?: boolean;
+    }): {
+        contextText: LlamaText;
+        stopGenerationTriggers: LlamaText[];
+    };
+}
+type ChatTemplate = `${`${string}{{systemPrompt}}` | ""}${string}{{history}}${string}{{completion}}${string}`;
+type ChatHistoryTemplate = `${string}{{roleName}}${string}{{message}}${string}`;
+type ChatHistoryFunctionCallMessageTemplate = [
+    call: `${string}{{functionName}}${string}{{functionParams}}${string}`,
+    result: `${string}{{functionCallResult}}${string}`
+];
+export {};

package/dist/TemplateChatWrapper.js

@@ -0,0 +1,239 @@
+import { BuiltinSpecialToken, LlamaText, SpecialToken } from "./utils/LlamaText.js";
+import { ChatWrapper } from "./ChatWrapper.js";
+import { parseTextTemplate } from "./utils/parseTextTemplate.js";
+/**
+ * A chat wrapper based on a simple template.
+ *
+ * Example usage:
+ * ```typescript
+ * const chatWrapper = new TemplateChatWrapper({
+ *     template: "{{systemPrompt}}\n{{history}}model:{{completion}}\nuser:",
+ *     historyTemplate: "{{roleName}}: {{message}}\n",
+ *     modelRoleName: "model",
+ *     userRoleName: "user",
+ *     systemRoleName: "system", // optional
+ *     // functionCallMessageTemplate: [ // optional
+ *     //     "[[call: {{functionName}}({{functionParams}})]]",
+ *     //     " [[result: {{functionCallResult}}]]"
+ *     // ]
+ * });
+ * ```
+ *
+ * **`{{systemPrompt}}`** is optional and is replaced with the first system message
+ * (when is does, that system message is not included in the history).
+ *
+ * **`{{history}}`** is replaced with the chat history.
+ * Each message in the chat history is converted using template passed to `historyTemplate`, and all messages are joined together.
+ *
+ * **`{{completion}}`** is where the model's response is generated.
+ * The text that comes after `{{completion}}` is used to determine when the model has finished generating the response,
+ * and thus is mandatory.
+ *
+ * **`functionCallMessageTemplate`** is used to specify the format in which functions can be called by the model and
+ * how their results are fed to the model after the function call.
+ */
+export class TemplateChatWrapper extends ChatWrapper {
+    wrapperName = "Template";
+    settings;
+    template;
+    historyTemplate;
+    modelRoleName;
+    userRoleName;
+    systemRoleName;
+    joinAdjacentMessagesOfTheSameType;
+    /** @internal */ _parsedChatTemplate;
+    /** @internal */ _parsedChatHistoryTemplate;
+    constructor({ template, historyTemplate, modelRoleName, userRoleName, systemRoleName = "System", functionCallMessageTemplate, joinAdjacentMessagesOfTheSameType = true }) {
+        super();
+        this.template = template;
+        this.historyTemplate = historyTemplate;
+        this.modelRoleName = modelRoleName;
+        this.userRoleName = userRoleName;
+        this.systemRoleName = systemRoleName;
+        this.joinAdjacentMessagesOfTheSameType = joinAdjacentMessagesOfTheSameType;
+        this._parsedChatTemplate = parseChatTemplate(template);
+        this._parsedChatHistoryTemplate = parseChatHistoryTemplate(historyTemplate);
+        this.settings = {
+            ...super.settings,
+            functions: parseFunctionCallMessageTemplate(functionCallMessageTemplate) ?? ChatWrapper.defaultSetting.functions
+        };
+    }
+    generateContextText(history, { availableFunctions, documentFunctionParams } = {}) {
+        const historyWithFunctions = this.addAvailableFunctionsSystemMessageToHistory(history, availableFunctions, {
+            documentParams: documentFunctionParams
+        });
+        const resultItems = [];
+        let systemTexts = [];
+        let userTexts = [];
+        let modelTexts = [];
+        let currentAggregateFocus = null;
+        function flush() {
+            if (systemTexts.length > 0 || userTexts.length > 0 || modelTexts.length > 0)
+                resultItems.push({
+                    system: systemTexts.join("\n\n"),
+                    user: userTexts.join("\n\n"),
+                    model: modelTexts.join("\n\n")
+                });
+            systemTexts = [];
+            userTexts = [];
+            modelTexts = [];
+        }
+        const getHistoryItem = (role, text, prefix) => {
+            const { roleNamePrefix, messagePrefix, messageSuffix } = this._parsedChatHistoryTemplate;
+            return LlamaText([
+                new SpecialToken((prefix ?? "") + roleNamePrefix + role + messagePrefix),
+                text,
+                new SpecialToken(messageSuffix)
+            ]);
+        };
+        for (const item of historyWithFunctions) {
+            if (item.type === "system") {
+                if (!this.joinAdjacentMessagesOfTheSameType || currentAggregateFocus !== "system")
+                    flush();
+                currentAggregateFocus = "system";
+                systemTexts.push(item.text);
+            }
+            else if (item.type === "user") {
+                if (!this.joinAdjacentMessagesOfTheSameType || (currentAggregateFocus !== "system" && currentAggregateFocus !== "user"))
+                    flush();
+                currentAggregateFocus = "user";
+                userTexts.push(item.text);
+            }
+            else if (item.type === "model") {
+                if (!this.joinAdjacentMessagesOfTheSameType)
+                    flush();
+                currentAggregateFocus = "model";
+                modelTexts.push(this.generateModelResponseText(item.response));
+            }
+        }
+        flush();
+        const contextText = LlamaText(resultItems.map(({ system, user, model }, index) => {
+            const isFirstItem = index === 0;
+            const isLastItem = index === resultItems.length - 1;
+            const res = LlamaText([
+                isFirstItem
+                    ? system.length === 0
+                        ? new SpecialToken((this._parsedChatTemplate.systemPromptPrefix ?? "") + this._parsedChatTemplate.historyPrefix)
+                        : this._parsedChatTemplate.systemPromptPrefix != null
+                            ? LlamaText([
+                                new SpecialToken(this._parsedChatTemplate.systemPromptPrefix),
+                                system,
+                                new SpecialToken(this._parsedChatTemplate.historyPrefix)
+                            ])
+                            : getHistoryItem("system", system, this._parsedChatTemplate.historyPrefix)
+                    : system.length === 0
+                        ? LlamaText([])
+                        : getHistoryItem("system", system),
+                user.length === 0
+                    ? LlamaText([])
+                    : getHistoryItem("user", user),
+                model.length === 0
+                    ? LlamaText([])
+                    : !isLastItem
+                        ? getHistoryItem("model", model)
+                        : LlamaText([
+                            new SpecialToken(this._parsedChatTemplate.completionPrefix),
+                            model
+                        ])
+            ]);
+            return LlamaText(res.values.reduce((res, value) => {
+                if (value instanceof SpecialToken) {
+                    const lastItem = res[res.length - 1];
+                    if (lastItem == null || !(lastItem instanceof SpecialToken))
+                        return res.concat([value]);
+                    return res.slice(0, -1).concat([
+                        new SpecialToken(lastItem.value + value.value)
+                    ]);
+                }
+                return res.concat([value]);
+            }, []));
+        }));
+        return {
+            contextText,
+            stopGenerationTriggers: [
+                LlamaText(new BuiltinSpecialToken("EOS")),
+                LlamaText(this._parsedChatTemplate.completionSuffix),
+                LlamaText(new SpecialToken(this._parsedChatTemplate.completionSuffix))
+            ]
+        };
+    }
+}
+function parseFunctionCallMessageTemplate(template) {
+    if (template == null)
+        return null;
+    const [functionCallTemplate, functionCallResultTemplate] = template;
+    if (functionCallTemplate == null || functionCallResultTemplate == null)
+        throw new Error("Both function call and function call result templates are required");
+    const parsedFunctionCallTemplate = parseTextTemplate(functionCallTemplate, [{
+            text: "{{functionName}}",
+            key: "functionName"
+        }, {
+            text: "{{functionParams}}",
+            key: "functionParams"
+        }]);
+    const parsedFunctionCallResultTemplate = parseTextTemplate(functionCallResultTemplate, [{
+            text: "{{functionCallResult}}",
+            key: "functionCallResult"
+        }]);
+    const callPrefix = parsedFunctionCallTemplate.functionName.prefix;
+    const callParamsPrefix = parsedFunctionCallTemplate.functionParams.prefix;
+    const callSuffix = parsedFunctionCallTemplate.functionParams.suffix;
+    const resultPrefix = parsedFunctionCallResultTemplate.functionCallResult.prefix;
+    const resultSuffix = parsedFunctionCallResultTemplate.functionCallResult.suffix;
+    if (callPrefix.length === 0)
+        throw new Error('Function call template must have text before "{{functionName}}"');
+    if (callSuffix.length === 0)
+        throw new Error('Function call template must have text after "{{functionParams}}"');
+    if (resultPrefix.length === 0)
+        throw new Error('Function call result template must have text before "{{functionCallResult}}"');
+    if (resultSuffix.length === 0)
+        throw new Error('Function call result template must have text after "{{functionCallResult}}"');
+    return {
+        call: {
+            optionalPrefixSpace: true,
+            prefix: callPrefix,
+            paramsPrefix: callParamsPrefix,
+            suffix: callSuffix
+        },
+        result: {
+            prefix: resultPrefix,
+            suffix: resultSuffix
+        }
+    };
+}
+function parseChatTemplate(template) {
+    const parsedTemplate = parseTextTemplate(template, [{
+            text: "{{systemPrompt}}",
+            key: "systemPrompt",
+            optional: true
+        }, {
+            text: "{{history}}",
+            key: "history"
+        }, {
+            text: "{{completion}}",
+            key: "completion"
+        }]);
+    if (parsedTemplate.completion.suffix.length == 0)
+        throw new Error('Chat template must have text after "{{completion}}"');
+    return {
+        systemPromptPrefix: parsedTemplate.systemPrompt?.prefix ?? null,
+        historyPrefix: parsedTemplate.history.prefix,
+        completionPrefix: parsedTemplate.completion.prefix,
+        completionSuffix: parsedTemplate.completion.suffix
+    };
+}
+function parseChatHistoryTemplate(template) {
+    const parsedTemplate = parseTextTemplate(template, [{
+            text: "{{roleName}}",
+            key: "roleName"
+        }, {
+            text: "{{message}}",
+            key: "message"
+        }]);
+    return {
+        roleNamePrefix: parsedTemplate.roleName.prefix,
+        messagePrefix: parsedTemplate.message.prefix,
+        messageSuffix: parsedTemplate.message.suffix
+    };
+}
+//# sourceMappingURL=TemplateChatWrapper.js.map

package/dist/TemplateChatWrapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TemplateChatWrapper.js","sourceRoot":"","sources":["../src/TemplateChatWrapper.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,mBAAmB,EAAE,SAAS,EAAkB,YAAY,EAAC,MAAM,sBAAsB,CAAC;AAClG,OAAO,EAAC,WAAW,EAAsB,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AAE/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,OAAO,mBAAoB,SAAQ,WAAW;IAChC,WAAW,GAAG,UAAU,CAAC;IAChB,QAAQ,CAAsB;IAEvC,QAAQ,CAAe;IACvB,eAAe,CAAsB;IACrC,aAAa,CAAS;IACtB,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,iCAAiC,CAAU;IAE3D,gBAAgB,CAAkB,mBAAmB,CAAuC;IAC5F,gBAAgB,CAAkB,0BAA0B,CAA8C;IAE1G,YAAmB,EACf,QAAQ,EACR,eAAe,EACf,aAAa,EACb,YAAY,EACZ,cAAc,GAAG,QAAQ,EACzB,2BAA2B,EAC3B,iCAAiC,GAAG,IAAI,EAS3C;QACG,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,iCAAiC,GAAG,iCAAiC,CAAC;QAE3E,IAAI,CAAC,mBAAmB,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,CAAC,0BAA0B,GAAG,wBAAwB,CAAC,eAAe,CAAC,CAAC;QAE5E,IAAI,CAAC,QAAQ,GAAG;YACZ,GAAG,KAAK,CAAC,QAAQ;YACjB,SAAS,EAAE,gCAAgC,CAAC,2BAA2B,CAAC,IAAI,WAAW,CAAC,cAAc,CAAC,SAAS;SACnH,CAAC;IACN,CAAC;IAEe,mBAAmB,CAAC,OAAmC,EAAE,EAAC,kBAAkB,EAAE,sBAAsB,KAGhH,EAAE;QAIF,MAAM,oBAAoB,GAAG,IAAI,CAAC,2CAA2C,CAAC,OAAO,EAAE,kBAAkB,EAAE;YACvG,cAAc,EAAE,sBAAsB;SACzC,CAAC,CAAC;QAEH,MAAM,WAAW,GAIZ,EAAE,CAAC;QAER,IAAI,WAAW,GAAa,EAAE,CAAC;QAC/B,IAAI,SAAS,GAAa,EAAE,CAAC;QAC7B,IAAI,UAAU,GAAa,EAAE,CAAC;QAC9B,IAAI,qBAAqB,GAAuC,IAAI,CAAC;QAErE,SAAS,KAAK;YACV,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;gBACvE,WAAW,CAAC,IAAI,CAAC;oBACb,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC;oBAChC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;oBAC5B,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;iBACjC,CAAC,CAAC;YAEP,WAAW,GAAG,EAAE,CAAC;YACjB,SAAS,GAAG,EAAE,CAAC;YACf,UAAU,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,MAAM,cAAc,GAAG,CAAC,IAAiC,EAAE,IAAY,EAAE,MAAsB,EAAE,EAAE;YAC/F,MAAM,EAAC,cAAc,EAAE,aAAa,EAAE,aAAa,EAAC,GAAG,IAAI,CAAC,0BAA0B,CAAC;YACvF,OAAO,SAAS,CAAC;gBACb,IAAI,YAAY,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,GAAG,cAAc,GAAG,IAAI,GAAG,aAAa,CAAC;gBACxE,IAAI;gBACJ,IAAI,YAAY,CAAC,aAAa,CAAC;aAClC,CAAC,CAAC;QACP,CAAC,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,oBAAoB,EAAE;YACrC,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE;gBACxB,IAAI,CAAC,IAAI,CAAC,iCAAiC,IAAI,qBAAqB,KAAK,QAAQ;oBAC7E,KAAK,EAAE,CAAC;gBAEZ,qBAAqB,GAAG,QAAQ,CAAC;gBACjC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC/B;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;gBAC7B,IAAI,CAAC,IAAI,CAAC,iCAAiC,IAAI,CAAC,qBAAqB,KAAK,QAAQ,IAAI,qBAAqB,KAAK,MAAM,CAAC;oBACnH,KAAK,EAAE,CAAC;gBAEZ,qBAAqB,GAAG,MAAM,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC7B;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE;gBAC9B,IAAI,CAAC,IAAI,CAAC,iCAAiC;oBACvC,KAAK,EAAE,CAAC;gBAEZ,qBAAqB,GAAG,OAAO,CAAC;gBAChC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;aAClE;SACJ;QAED,KAAK,EAAE,CAAC;QAER,MAAM,WAAW,GAAG,SAAS,CACzB,WAAW,CAAC,GAAG,CAAC,CAAC,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAC,EAAE,KAAK,EAAE,EAAE;YAC7C,MAAM,WAAW,GAAG,KAAK,KAAK,CAAC,CAAC;YAChC,MAAM,UAAU,GAAG,KAAK,KAAK,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;YAEpD,MAAM,GAAG,GAAG,SAAS,CAAC;gBAClB,WAAW;oBACP,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;wBACjB,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC;wBAChH,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,IAAI,IAAI;4BACjD,CAAC,CAAC,SAAS,CAAC;gCACR,IAAI,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,CAAC;gCAC7D,MAAM;gCACN,IAAI,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC;6BAC3D,CAAC;4BACF,CAAC,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC;oBAClF,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;wBACjB,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;wBACf,CAAC,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAG1C,IAAI,CAAC,MAAM,KAAK,CAAC;oBACb,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;oBACf,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC;gBAElC,KAAK,CAAC,MAAM,KAAK,CAAC;oBACd,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;oBACf,CAAC,CAAC,CAAC,UAAU;wBACT,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC;wBAChC,CAAC,CAAC,SAAS,CAAC;4BACR,IAAI,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC;4BAC3D,KAAK;yBACR,CAAC;aACb,CAAC,CAAC;YAEH,OAAO,SAAS,CACZ,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBAC7B,IAAI,KAAK,YAAY,YAAY,EAAE;oBAC/B,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAErC,IAAI,QAAQ,IAAI,IAAI,IAAI,CAAC,CAAC,QAAQ,YAAY,YAAY,CAAC;wBACvD,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;oBAE/B,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;wBAC3B,IAAI,YAAY,CAAC,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;qBACjD,CAAC,CAAC;iBACN;gBAED,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/B,CAAC,EAAE,EAAsB,CAAC,CAC7B,CAAC;QACN,CAAC,CAAC,CACL,CAAC;QAEF,OAAO;YACH,WAAW;YACX,sBAAsB,EAAE;gBACpB,SAAS,CAAC,IAAI,mBAAmB,CAAC,KAAK,CAAC,CAAC;gBACzC,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC;gBACpD,SAAS,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;aACzE;SACJ,CAAC;IACN,CAAC;CACJ;AAUD,SAAS,gCAAgC,CAAC,QAAiD;IACvF,IAAI,QAAQ,IAAI,IAAI;QAChB,OAAO,IAAI,CAAC;IAEhB,MAAM,CAAC,oBAAoB,EAAE,0BAA0B,CAAC,GAAG,QAAQ,CAAC;IAEpE,IAAI,oBAAoB,IAAI,IAAI,IAAI,0BAA0B,IAAI,IAAI;QAClE,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IAE1F,MAAM,0BAA0B,GAAG,iBAAiB,CAAC,oBAAoB,EAAE,CAAC;YACxE,IAAI,EAAE,kBAAkB;YACxB,GAAG,EAAE,cAAc;SACtB,EAAE;YACC,IAAI,EAAE,oBAAoB;YAC1B,GAAG,EAAE,gBAAgB;SACxB,CAAC,CAAC,CAAC;IACJ,MAAM,gCAAgC,GAAG,iBAAiB,CAAC,0BAA0B,EAAE,CAAC;YACpF,IAAI,EAAE,wBAAwB;YAC9B,GAAG,EAAE,oBAAoB;SAC5B,CAAC,CAAC,CAAC;IAEJ,MAAM,UAAU,GAAG,0BAA0B,CAAC,YAAY,CAAC,MAAM,CAAC;IAClE,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,cAAc,CAAC,MAAM,CAAC;IAC1E,MAAM,UAAU,GAAG,0BAA0B,CAAC,cAAc,CAAC,MAAM,CAAC;IAEpE,MAAM,YAAY,GAAG,gCAAgC,CAAC,kBAAkB,CAAC,MAAM,CAAC;IAChF,MAAM,YAAY,GAAG,gCAAgC,CAAC,kBAAkB,CAAC,MAAM,CAAC;IAEhF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IAEvF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IAExF,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAEpG,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;IAEnG,OAAO;QACH,IAAI,EAAE;YACF,mBAAmB,EAAE,IAAI;YACzB,MAAM,EAAE,UAAU;YAClB,YAAY,EAAE,gBAAgB;YAC9B,MAAM,EAAE,UAAU;SACrB;QACD,MAAM,EAAE;YACJ,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,YAAY;SACvB;KACJ,CAAC;AACN,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAsB;IAM7C,MAAM,cAAc,GAAG,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YAChD,IAAI,EAAE,kBAAkB;YACxB,GAAG,EAAE,cAAc;YACnB,QAAQ,EAAE,IAAI;SACjB,EAAE;YACC,IAAI,EAAE,aAAa;YACnB,GAAG,EAAE,SAAS;SACjB,EAAE;YACC,IAAI,EAAE,gBAAgB;YACtB,GAAG,EAAE,YAAY;SACpB,CAAC,CAAC,CAAC;IAEJ,IAAI,cAAc,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IAE3E,OAAO;QACH,kBAAkB,EAAE,cAAc,CAAC,YAAY,EAAE,MAAM,IAAI,IAAI;QAC/D,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC,MAAM;QAC5C,gBAAgB,EAAE,cAAc,CAAC,UAAU,CAAC,MAAM;QAClD,gBAAgB,EAAE,cAAc,CAAC,UAAU,CAAC,MAAM;KACrD,CAAC;AACN,CAAC;AAED,SAAS,wBAAwB,CAAC,QAA6B;IAK3D,MAAM,cAAc,GAAG,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YAChD,IAAI,EAAE,cAAc;YACpB,GAAG,EAAE,UAAU;SAClB,EAAE;YACC,IAAI,EAAE,aAAa;YACnB,GAAG,EAAE,SAAS;SACjB,CAAC,CAAC,CAAC;IAEJ,OAAO;QACH,cAAc,EAAE,cAAc,CAAC,QAAQ,CAAC,MAAM;QAC9C,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC,MAAM;QAC5C,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC,MAAM;KAC/C,CAAC;AACN,CAAC"}

package/dist/bindings/AddonTypes.d.ts

@@ -33,6 +33,8 @@ export type BindingModule = {
         total: number;
         used: number;
     };
+    getGpuType(): "cuda" | "vulkan" | "metal" | undefined;
+    init(): void;
 };
 export type AddonModel = {
     dispose(): void;

package/dist/bindings/Llama.d.ts

@@ -1,8 +1,7 @@
 import { LlamaLogLevel } from "./types.js";
 export declare class Llama {
     private constructor();
-    get metal(): boolean;
-    get cuda(): boolean;
+    get gpu(): false | "cuda" | "vulkan" | "metal";
     get logLevel(): LlamaLogLevel;
     set logLevel(value: LlamaLogLevel);
     get logger(): (level: LlamaLogLevel, message: string) => void;

package/dist/bindings/Llama.js

@@ -13,8 +13,7 @@ const addonLogLevelToLlamaLogLevel = new Map([...LlamaLogLevelToAddonLogLevel.en
 const defaultLogLevel = 5;
 export class Llama {
     /** @internal */ _bindings;
-    /** @internal */ _metal;
-    /** @internal */ _cuda;
+    /** @internal */ _gpu;
     /** @internal */ _buildType;
     /** @internal */ _cmakeOptions;
     /** @internal */ _llamaCppRelease;
@@ -26,10 +25,9 @@ export class Llama {
     /** @internal */ _previousLog = null;
     /** @internal */ _previousLogLevel = null;
     /** @internal */ _nextLogNeedNewLine = false;
-    constructor({ bindings, metal, cuda, logLevel, logger, buildType, cmakeOptions, llamaCppRelease }) {
+    constructor({ bindings, logLevel, logger, buildType, cmakeOptions, llamaCppRelease, _skipLlamaInit = false }) {
         this._bindings = bindings;
-        this._metal = metal;
-        this._cuda = cuda;
+        this._gpu = bindings.getGpuType() ?? false;
         this._logLevel = logLevel ?? LlamaLogLevel.debug;
         this._logger = logger;
         this._buildType = buildType;
@@ -40,14 +38,13 @@ export class Llama {
         });
         this._dispatchPendingLogMicrotask = this._dispatchPendingLogMicrotask.bind(this);
         this._onAddonLog = this._onAddonLog.bind(this);
+        if (!_skipLlamaInit)
+            this._bindings.init();
         this._bindings.setLogger(this._onAddonLog);
         this._bindings.setLoggerLogLevel(LlamaLogLevelToAddonLogLevel.get(this._logLevel) ?? defaultLogLevel);
     }
-    get metal() {
-        return this._metal;
-    }
-    get cuda() {
-        return this._cuda;
+    get gpu() {
+        return this._gpu;
     }
     get logLevel() {
         return this._logLevel;
@@ -151,19 +148,18 @@ export class Llama {
         this._previousLogLevel = level;
     }
     /** @internal */
-    static async _create({ bindings, buildType, buildMetadata, logLevel, logger }) {
+    static async _create({ bindings, buildType, buildMetadata, logLevel, logger, skipLlamaInit = false }) {
         return new Llama({
             bindings,
             buildType,
-            metal: buildMetadata.buildOptions.computeLayers.metal,
-            cuda: buildMetadata.buildOptions.computeLayers.cuda,
             cmakeOptions: buildMetadata.buildOptions.customCmakeOptions,
             llamaCppRelease: {
                 repo: buildMetadata.buildOptions.llamaCpp.repo,
                 release: buildMetadata.buildOptions.llamaCpp.release
             },
             logLevel,
-            logger
+            logger,
+            _skipLlamaInit: skipLlamaInit
         });
     }
     static defaultConsoleLogger(level, message) {

package/dist/bindings/Llama.js.map

@@ -1 +1 @@
-{"version":3,"file":"Llama.js","sourceRoot":"","sources":["../../src/bindings/Llama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,mBAAmB,EAAC,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAAoB,aAAa,EAAC,MAAM,YAAY,CAAC;AAE5D,MAAM,4BAA4B,GAAuC,IAAI,GAAG,CAAC;IAC7E,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3B,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC;IACxB,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC;IACxB,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;IACvB,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;IACvB,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC;CAC3B,CAAC,CAAC;AACH,MAAM,4BAA4B,GAAuC,IAAI,GAAG,CAC5E,CAAC,GAAG,4BAA4B,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAClF,CAAC;AACF,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,MAAM,OAAO,KAAK;IACd,gBAAgB,CAAiB,SAAS,CAAgB;IAC1D,gBAAgB,CAAkB,MAAM,CAAU;IAClD,gBAAgB,CAAkB,KAAK,CAAU;IACjD,gBAAgB,CAAkB,UAAU,CAA4B;IACxE,gBAAgB,CAAkB,aAAa,CAAmC;IAClF,gBAAgB,CAAkB,gBAAgB,CAGhD;IACF,gBAAgB,CAAS,OAAO,CAAoD;IACpF,gBAAgB,CAAS,SAAS,CAAgB;IAClD,gBAAgB,CAAS,WAAW,GAAkB,IAAI,CAAC;IAC3D,gBAAgB,CAAS,gBAAgB,GAAyB,IAAI,CAAC;IACvE,gBAAgB,CAAS,4BAA4B,GAAW,CAAC,CAAC;IAClE,gBAAgB,CAAS,YAAY,GAAkB,IAAI,CAAC;IAC5D,gBAAgB,CAAS,iBAAiB,GAAyB,IAAI,CAAC;IACxE,gBAAgB,CAAS,mBAAmB,GAAY,KAAK,CAAC;IAE9D,YAAoB,EAChB,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,eAAe,EAapF;QACG,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,SAAS,GAAG,QAAQ,IAAI,aAAa,CAAC,KAAK,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,EAAC,GAAG,YAAY,EAAC,CAAC,CAAC;QACtD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC;YAClC,IAAI,EAAE,eAAe,CAAC,IAAI;YAC1B,OAAO,EAAE,eAAe,CAAC,OAAO;SACnC,CAAC,CAAC;QAEH,IAAI,CAAC,4BAA4B,GAAG,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE/C,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,4BAA4B,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,eAAe,CAAC,CAAC;IAC1G,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED,IAAW,IAAI;QACX,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ,CAAC,KAAoB;QACpC,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS;YACxB,OAAO;QAEX,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,4BAA4B,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,CAAC;QAC7F,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAW,MAAM,CAAC,KAAsD;QACpE,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,KAAK,KAAK,CAAC,oBAAoB;YACpC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;IACzC,CAAC;IAED,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC;IACvC,CAAC;IAEM,YAAY;QACf,MAAM,EAAC,KAAK,EAAE,IAAI,EAAC,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC;QAEtD,OAAO;YACH,KAAK;YACL,IAAI;YACJ,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;SAClC,CAAC;IACN,CAAC;IAED,gBAAgB;IACR,WAAW,CAAC,KAAa,EAAE,OAAe;QAC9C,MAAM,aAAa,GAAG,4BAA4B,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC;QAErF,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,IAAI,aAAa,EAAE;YACrG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;SAC3B;QAED,MAAM,aAAa,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC;QAEzD,MAAM,gBAAgB,GAAG,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,gBAAgB,GAAG,CAAC;YACnC,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,gBAAgB,GAAG,CAAC;YAChC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QAEhD,IAAI,UAAU,KAAK,EAAE;YACjB,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;QAEhD,IAAI,OAAO,KAAK,EAAE,EAAE;YAChB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,aAAa,CAAC;YAEtC,cAAc,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAClD,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACvC;;YACG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAChC,CAAC;IAED,gBAAgB;IACR,4BAA4B;QAChC,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,4BAA4B,KAAK,CAAC;YACvC,OAAO;QAEX,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,EAAE;YAC3D,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;SAC3B;IACL,CAAC;IAED,gBAAgB;IACR,WAAW,CAAC,KAAoB,EAAE,OAAe;QACrD,+FAA+F;QAC/F,2CAA2C;QAC3C,IAAI,oBAAoB,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,oBAAoB,EAAE;YAC9E,IAAI,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,KAAK,KAAK,IAAI,CAAC,iBAAiB,EAAE;gBAC7E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;aACjC;iBAAM;gBACH,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;gBAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aACpF;SACJ;aAAM;YACH,IAAI,IAAI,CAAC,mBAAmB,EAAE;gBAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC3B,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;aACpC;YAED,IAAI;gBACA,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;aAChC;YAAC,OAAO,GAAG,EAAE;gBACV,sFAAsF;aACzF;SACJ;QAED,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;QAC5B,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;IACnC,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EACxB,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAOvD;QACG,OAAO,IAAI,KAAK,CAAC;YACb,QAAQ;YACR,SAAS;YACT,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK;YACrD,IAAI,EAAE,aAAa,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI;YACnD,YAAY,EAAE,aAAa,CAAC,YAAY,CAAC,kBAAkB;YAC3D,eAAe,EAAE;gBACb,IAAI,EAAE,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI;gBAC9C,OAAO,EAAE,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO;aACvD;YACD,QAAQ;YACR,MAAM;SACT,CAAC,CAAC;IACP,CAAC;IAEM,MAAM,CAAC,oBAAoB,CAAC,KAAoB,EAAE,OAAe;QACpE,QAAQ,KAAK,EAAE;YACX,KAAK,aAAa,CAAC,QAAQ;gBACvB,MAAM;YACV,KAAK,aAAa,CAAC,KAAK;gBACpB,oEAAoE;gBACpE,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzE,MAAM;YACV,KAAK,aAAa,CAAC,KAAK;gBACpB,oEAAoE;gBACpE,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzE,MAAM;YACV,KAAK,aAAa,CAAC,IAAI;gBACnB,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzE,MAAM;YACV,KAAK,aAAa,CAAC,IAAI;gBACnB,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzE,MAAM;YACV,KAAK,aAAa,CAAC,KAAK;gBACpB,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1E,MAAM;YACV;gBACI,KAAK,CAAC,KAAqB,CAAC,CAAC;gBAC7B,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,mBAAmB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC7G,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAC/E;IAEL,CAAC;CACJ;AAED,SAAS,mBAAmB,CAAC,KAAoB;IAC7C,QAAQ,KAAK,EAAE;QACX,KAAK,aAAa,CAAC,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,WAAW,CAAC;QACtD,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,SAAS,CAAC;QACjD,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC;QAC3C,KAAK,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC;QAC7C,KAAK,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,WAAW,CAAC;QAClD,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC;QAC7C;YACI,KAAK,CAAC,KAAqB,CAAC,CAAC;YAC7B,OAAO,KAAK,CAAC,WAAW,CAAC;KAChC;AACL,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe,EAAE,KAAkC;IAC9E,OAAO,mBAAmB,EAAE,GAAG,CAC3B,OAAO;SACF,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SACxB,IAAI,CAAC,IAAI,GAAG,mBAAmB,EAAE,CAAC,CAC1C,CAAC;AACN,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAsB;IAChD,IAAI,OAAO,IAAI,IAAI;QACf,OAAO,KAAK,CAAC;IAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACrC,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,GAAG;YAClB,OAAO,KAAK,CAAC;KACpB;IAED,OAAO,IAAI,CAAC;AAChB,CAAC"}
+{"version":3,"file":"Llama.js","sourceRoot":"","sources":["../../src/bindings/Llama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,mBAAmB,EAAC,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAA8B,aAAa,EAAC,MAAM,YAAY,CAAC;AAEtE,MAAM,4BAA4B,GAAuC,IAAI,GAAG,CAAC;IAC7E,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3B,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC;IACxB,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC;IACxB,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;IACvB,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;IACvB,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC;CAC3B,CAAC,CAAC;AACH,MAAM,4BAA4B,GAAuC,IAAI,GAAG,CAC5E,CAAC,GAAG,4BAA4B,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAClF,CAAC;AACF,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,MAAM,OAAO,KAAK;IACd,gBAAgB,CAAiB,SAAS,CAAgB;IAC1D,gBAAgB,CAAkB,IAAI,CAAW;IACjD,gBAAgB,CAAkB,UAAU,CAA4B;IACxE,gBAAgB,CAAkB,aAAa,CAAmC;IAClF,gBAAgB,CAAkB,gBAAgB,CAGhD;IACF,gBAAgB,CAAS,OAAO,CAAoD;IACpF,gBAAgB,CAAS,SAAS,CAAgB;IAClD,gBAAgB,CAAS,WAAW,GAAkB,IAAI,CAAC;IAC3D,gBAAgB,CAAS,gBAAgB,GAAyB,IAAI,CAAC;IACvE,gBAAgB,CAAS,4BAA4B,GAAW,CAAC,CAAC;IAClE,gBAAgB,CAAS,YAAY,GAAkB,IAAI,CAAC;IAC5D,gBAAgB,CAAS,iBAAiB,GAAyB,IAAI,CAAC;IACxE,gBAAgB,CAAS,mBAAmB,GAAY,KAAK,CAAC;IAE9D,YAAoB,EAChB,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,GAAG,KAAK,EAY/F;QACG,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,UAAU,EAAE,IAAI,KAAK,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,QAAQ,IAAI,aAAa,CAAC,KAAK,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,EAAC,GAAG,YAAY,EAAC,CAAC,CAAC;QACtD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC;YAClC,IAAI,EAAE,eAAe,CAAC,IAAI;YAC1B,OAAO,EAAE,eAAe,CAAC,OAAO;SACnC,CAAC,CAAC;QAEH,IAAI,CAAC,4BAA4B,GAAG,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE/C,IAAI,CAAC,cAAc;YACf,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAE1B,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,4BAA4B,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,eAAe,CAAC,CAAC;IAC1G,CAAC;IAED,IAAW,GAAG;QACV,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ,CAAC,KAAoB;QACpC,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS;YACxB,OAAO;QAEX,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,4BAA4B,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,CAAC;QAC7F,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAW,MAAM,CAAC,KAAsD;QACpE,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,KAAK,KAAK,CAAC,oBAAoB;YACpC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;IACzC,CAAC;IAED,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC;IACvC,CAAC;IAEM,YAAY;QACf,MAAM,EAAC,KAAK,EAAE,IAAI,EAAC,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC;QAEtD,OAAO;YACH,KAAK;YACL,IAAI;YACJ,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;SAClC,CAAC;IACN,CAAC;IAED,gBAAgB;IACR,WAAW,CAAC,KAAa,EAAE,OAAe;QAC9C,MAAM,aAAa,GAAG,4BAA4B,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC;QAErF,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,IAAI,aAAa,EAAE;YACrG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;SAC3B;QAED,MAAM,aAAa,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC;QAEzD,MAAM,gBAAgB,GAAG,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,gBAAgB,GAAG,CAAC;YACnC,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,gBAAgB,GAAG,CAAC;YAChC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QAEhD,IAAI,UAAU,KAAK,EAAE;YACjB,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;QAEhD,IAAI,OAAO,KAAK,EAAE,EAAE;YAChB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,aAAa,CAAC;YAEtC,cAAc,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAClD,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACvC;;YACG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAChC,CAAC;IAED,gBAAgB;IACR,4BAA4B;QAChC,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,4BAA4B,KAAK,CAAC;YACvC,OAAO;QAEX,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,EAAE;YAC3D,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;SAC3B;IACL,CAAC;IAED,gBAAgB;IACR,WAAW,CAAC,KAAoB,EAAE,OAAe;QACrD,+FAA+F;QAC/F,2CAA2C;QAC3C,IAAI,oBAAoB,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,oBAAoB,EAAE;YAC9E,IAAI,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,KAAK,KAAK,IAAI,CAAC,iBAAiB,EAAE;gBAC7E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;aACjC;iBAAM;gBACH,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;gBAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aACpF;SACJ;aAAM;YACH,IAAI,IAAI,CAAC,mBAAmB,EAAE;gBAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC3B,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;aACpC;YAED,IAAI;gBACA,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;aAChC;YAAC,OAAO,GAAG,EAAE;gBACV,sFAAsF;aACzF;SACJ;QAED,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;QAC5B,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;IACnC,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EACxB,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,GAAG,KAAK,EAQ9E;QACG,OAAO,IAAI,KAAK,CAAC;YACb,QAAQ;YACR,SAAS;YACT,YAAY,EAAE,aAAa,CAAC,YAAY,CAAC,kBAAkB;YAC3D,eAAe,EAAE;gBACb,IAAI,EAAE,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI;gBAC9C,OAAO,EAAE,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO;aACvD;YACD,QAAQ;YACR,MAAM;YACN,cAAc,EAAE,aAAa;SAChC,CAAC,CAAC;IACP,CAAC;IAEM,MAAM,CAAC,oBAAoB,CAAC,KAAoB,EAAE,OAAe;QACpE,QAAQ,KAAK,EAAE;YACX,KAAK,aAAa,CAAC,QAAQ;gBACvB,MAAM;YACV,KAAK,aAAa,CAAC,KAAK;gBACpB,oEAAoE;gBACpE,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzE,MAAM;YACV,KAAK,aAAa,CAAC,KAAK;gBACpB,oEAAoE;gBACpE,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzE,MAAM;YACV,KAAK,aAAa,CAAC,IAAI;gBACnB,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzE,MAAM;YACV,KAAK,aAAa,CAAC,IAAI;gBACnB,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzE,MAAM;YACV,KAAK,aAAa,CAAC,KAAK;gBACpB,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1E,MAAM;YACV;gBACI,KAAK,CAAC,KAAqB,CAAC,CAAC;gBAC7B,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,mBAAmB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC7G,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAC/E;IAEL,CAAC;CACJ;AAED,SAAS,mBAAmB,CAAC,KAAoB;IAC7C,QAAQ,KAAK,EAAE;QACX,KAAK,aAAa,CAAC,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,WAAW,CAAC;QACtD,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,SAAS,CAAC;QACjD,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC;QAC3C,KAAK,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC;QAC7C,KAAK,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,WAAW,CAAC;QAClD,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC;QAC7C;YACI,KAAK,CAAC,KAAqB,CAAC,CAAC;YAC7B,OAAO,KAAK,CAAC,WAAW,CAAC;KAChC;AACL,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe,EAAE,KAAkC;IAC9E,OAAO,mBAAmB,EAAE,GAAG,CAC3B,OAAO;SACF,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SACxB,IAAI,CAAC,IAAI,GAAG,mBAAmB,EAAE,CAAC,CAC1C,CAAC;AACN,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAsB;IAChD,IAAI,OAAO,IAAI,IAAI;QACf,OAAO,KAAK,CAAC;IAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACrC,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,GAAG;YAClB,OAAO,KAAK,CAAC;KACpB;IAED,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/bindings/consts.d.ts

@@ -0,0 +1,2 @@
+import { BuildGpu } from "./types.js";
+export declare function getPrettyBuildGpuName(gpu: BuildGpu): string;

package/dist/bindings/consts.js

@@ -0,0 +1,11 @@
+const prettyBuildGpuNames = {
+    metal: "Metal",
+    cuda: "CUDA",
+    vulkan: "Vulkan"
+};
+export function getPrettyBuildGpuName(gpu) {
+    if (gpu == false)
+        return "no GPU";
+    return prettyBuildGpuNames[gpu] ?? ('"' + gpu + '"');
+}
+//# sourceMappingURL=consts.js.map

package/dist/bindings/consts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../src/bindings/consts.ts"],"names":[],"mappings":"AAEA,MAAM,mBAAmB,GAA6C;IAClE,KAAK,EAAE,OAAO;IACd,IAAI,EAAE,MAAM;IACZ,MAAM,EAAE,QAAQ;CACnB,CAAC;AAEF,MAAM,UAAU,qBAAqB,CAAC,GAAa;IAC/C,IAAI,GAAG,IAAI,KAAK;QACZ,OAAO,QAAQ,CAAC;IAEpB,OAAO,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;AACzD,CAAC"}

package/dist/bindings/getLlama.d.ts

@@ -2,16 +2,18 @@ import { LlamaLogLevel } from "./types.js";
 import { Llama } from "./Llama.js";
 export type LlamaOptions = {
     /**
-     * Toggle Metal support in llama.cpp.
-     * Only supported on macOS.
-     * Enabled by default on Apple Silicon Macs.
-     */
-    metal?: boolean;
-    /**
-     * Toggle CUDA support on llama.cpp.
-     * Disabled by default.
+     * The compute layer implementation type to use for llama.cpp.
+     * - **`"auto"`**: Automatically detect and use the best GPU available (Metal on macOS, and CUDA or Vulkan on Windows and Linux)
+     * - **`"metal"`**: Use Metal.
+     *   Only supported on macOS.
+     *   Enabled by default on Apple Silicon Macs.
+     * - **`"cuda"`**: Use CUDA.
+     * - **`"vulkan"`**: Use Vulkan.
+     * - **`false`**: Disable any GPU support and only use the CPU.
+     *
+     * `"auto"` by default.
      */
-    cuda?: boolean;
+    gpu?: "auto" | "metal" | "cuda" | "vulkan" | false;
     /**
      * Set the minimum log level for llama.cpp.
      * Defaults to "debug".
@@ -95,9 +97,9 @@ export declare const getLlamaFunctionName = "getLlama";
  * Defaults to prefer a prebuilt binary, and fallbacks to building from source if a prebuilt binary is not found.
  * Pass `"lastCliBuild"` to default to use the last successful build created using the `download` or `build` CLI commands if one exists.
  */
-export declare function getLlama(): Promise<Llama>;
-export declare function getLlama(options: LlamaOptions, lastBuildOptions?: never): Promise<Llama>;
+export declare function getLlama(options?: LlamaOptions): Promise<Llama>;
 export declare function getLlama(type: "lastBuild", lastBuildOptions?: LastBuildOptions): Promise<Llama>;
-export declare function getLlamaForOptions({ metal, cuda, logLevel, logger, build, cmakeOptions, existingPrebuiltBinaryMustMatchBuildOptions, usePrebuiltBinaries, progressLogs, skipDownload }: LlamaOptions, { updateLastBuildInfoOnCompile }?: {
+export declare function getLlamaForOptions({ gpu, logLevel, logger, build, cmakeOptions, existingPrebuiltBinaryMustMatchBuildOptions, usePrebuiltBinaries, progressLogs, skipDownload }: LlamaOptions, { updateLastBuildInfoOnCompile, skipLlamaInit }?: {
     updateLastBuildInfoOnCompile?: boolean;
+    skipLlamaInit?: boolean;
 }): Promise<Llama>;