node-llama-cpp 3.6.0 → 3.8.0

package/dist/utils/resolveModelFile.js

@@ -75,7 +75,7 @@ import { pushAll } from "./pushAll.js";
  * @returns The resolved model file path
  */
 export async function resolveModelFile(uriOrPath, optionsOrDirectory) {
-    const { directory, download = "auto", verify = false, fileName, headers, cli = true, onProgress, deleteTempFileOnCancel = true, parallel = 4, tokens, signal } = typeof optionsOrDirectory === "string"
+    const { directory, download = "auto", verify = false, fileName, headers, cli = true, onProgress, deleteTempFileOnCancel = true, parallel = 4, tokens, endpoints, signal } = typeof optionsOrDirectory === "string"
         ? { directory: optionsOrDirectory }
         : (optionsOrDirectory ?? {});
     const resolvedDirectory = directory || cliModelsDirectory;
@@ -83,7 +83,7 @@ export async function resolveModelFile(uriOrPath, optionsOrDirectory) {
     let resolvedVerify = verify ?? false;
     if (download === false)
         resolvedVerify = false;
-    const resolvedModelDestination = resolveModelDestination(uriOrPath);
+    const resolvedModelDestination = resolveModelDestination(uriOrPath, undefined, endpoints);
     if (resolvedModelDestination.type === "file") {
         const resolvedFilePath = path.resolve(resolvedDirectory, uriOrPath);
         if (await fs.pathExists(resolvedFilePath))
@@ -100,7 +100,7 @@ export async function resolveModelFile(uriOrPath, optionsOrDirectory) {
             pushAll(expectedFileNames, resolvedModelDestination.parsedUri.possibleFullFilenames);
     }
     else if (expectedFileNames.length === 0 && resolvedModelDestination.type === "url") {
-        const enforcedParsedUrl = resolveModelDestination(uriOrPath, true);
+        const enforcedParsedUrl = resolveModelDestination(uriOrPath, true, endpoints);
         if (enforcedParsedUrl != null && enforcedParsedUrl.type === "uri") {
             if (enforcedParsedUrl.parsedUri.type === "resolved")
                 expectedFileNames.push(enforcedParsedUrl.parsedUri.fullFilename);
@@ -140,7 +140,8 @@ export async function resolveModelFile(uriOrPath, optionsOrDirectory) {
         fileName: fileName || undefined,
         parallelDownloads: parallel,
         onProgress,
-        tokens
+        tokens,
+        endpoints
     });
     if (foundExpectedFilePath != null && downloader.totalFiles === 1 && await fs.pathExists(downloader.entrypointFilePath)) {
         const fileStats = await fs.stat(foundExpectedFilePath);

package/dist/utils/resolveModelFile.js.map

@@ -1 +1 @@
-{"version":3,"file":"resolveModelFile.js","sourceRoot":"","sources":["../../src/utils/resolveModelFile.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,kBAAkB,EAAC,MAAM,cAAc,CAAC;AAChD,OAAO,EAAC,eAAe,EAAC,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAC,qBAAqB,EAAC,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAC,uBAAuB,EAAC,MAAM,8BAA8B,CAAC;AAErE,OAAO,EAAC,qBAAqB,EAAC,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAC,qBAAqB,EAAC,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAC,cAAc,EAAC,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAgFrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAClC,SAAiB,EACjB,kBAAqD;IAErD,MAAM,EACF,SAAS,EACT,QAAQ,GAAG,MAAM,EACjB,MAAM,GAAG,KAAK,EACd,QAAQ,EACR,OAAO,EACP,GAAG,GAAG,IAAI,EACV,UAAU,EACV,sBAAsB,GAAG,IAAI,EAC7B,QAAQ,GAAG,CAAC,EACZ,MAAM,EACN,MAAM,EACT,GAAG,OAAO,kBAAkB,KAAK,QAAQ;QACtC,CAAC,CAAC,EAAC,SAAS,EAAE,kBAAkB,EAAC;QACjC,CAAC,CAAC,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;IAEjC,MAAM,iBAAiB,GAAG,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,WAAW,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7C,IAAI,cAAc,GAAG,MAAM,IAAI,KAAK,CAAC;IAErC,IAAI,QAAQ,KAAK,KAAK;QAClB,cAAc,GAAG,KAAK,CAAC;IAE3B,MAAM,wBAAwB,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAC;IAEpE,IAAI,wBAAwB,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3C,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAEpE,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC;YACrC,OAAO,gBAAgB,CAAC;QAE5B,MAAM,IAAI,KAAK,CAAC,2BAA2B,gBAAgB,GAAG,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,iBAAiB,GAAa,QAAQ,IAAI,IAAI;QAChD,CAAC,CAAC,CAAC,QAAQ,CAAC;QACZ,CAAC,CAAC,EAAE,CAAC;IAET,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,wBAAwB,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC5E,IAAI,wBAAwB,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU;YACtD,iBAAiB,CAAC,IAAI,CAAC,wBAAwB,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;;YAExE,OAAO,CAAC,iBAAiB,EAAE,wBAAwB,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;IAC7F,CAAC;SAAM,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,wBAAwB,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACnF,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACnE,IAAI,iBAAiB,IAAI,IAAI,IAAI,iBAAiB,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAChE,IAAI,iBAAiB,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU;gBAC/C,iBAAiB,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;;gBAEjE,OAAO,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;QACtF,CAAC;IACL,CAAC;IAED,MAAM,qBAAqB,GAAG,MAAM,4BAA4B,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;IAEvG,IAAI,qBAAqB,IAAI,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACnD,MAAM,YAAY,GAAG,qBAAqB,CAAC,qBAAqB,CAAC,CAAC;QAClE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,KAAK,qBAAqB;YACtE,OAAO,qBAAqB,CAAC;QAEjC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC;gBACvC,OAAO,YAAY,CAAC,CAAC,CAAE,CAAC;iBACvB,IAAI,QAAQ,KAAK,KAAK;gBACvB,MAAM,IAAI,KAAK,CAAC,0CAA0C,YAAY,CAAC,CAAC,CAAC,qCAAqC,CAAC,CAAC;QACxH,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;QACrB,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC,CAAE,CAAC,4BAA4B,CAAC,CAAC;QAEhI,MAAM,IAAI,KAAK,CAAC,4BAA4B,SAAS,SAAS,iBAAiB,4BAA4B,CAAC,CAAC;IACjH,CAAC;IAED,IAAI,MAAM,EAAE,OAAO;QACf,MAAM,MAAM,CAAC,MAAM,CAAC;IAExB,MAAM,UAAU,GAAG,MAAM,qBAAqB,CAAC;QAC3C,QAAQ,EAAE,wBAAwB,CAAC,IAAI,KAAK,KAAK;YAC7C,CAAC,CAAC,wBAAwB,CAAC,GAAG;YAC9B,CAAC,CAAC,wBAAwB,CAAC,GAAG;QAClC,OAAO,EAAE,iBAAiB;QAC1B,OAAO;QACP,eAAe,EAAE,WAAW;QAC5B,sBAAsB;QACtB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,QAAQ,IAAI,SAAS;QAC/B,iBAAiB,EAAE,QAAQ;QAC3B,UAAU;QACV,MAAM;KACT,CAAC,CAAC;IAEH,IAAI,qBAAqB,IAAI,IAAI,IAAI,UAAU,CAAC,UAAU,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACrH,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAEvD,IAAI,UAAU,CAAC,SAAS,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YAC1C,MAAM,UAAU,CAAC,MAAM,CAAC,EAAC,cAAc,EAAE,KAAK,EAAC,CAAC,CAAC;YACjD,OAAO,qBAAqB,CAAC;QACjC,CAAC;IACL,CAAC;IAED,IAAI,WAAW;QACX,OAAO,CAAC,IAAI,CAAC,kBAAkB,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,GAC3E,UAAU,CAAC,gBAAgB,IAAI,IAAI;YAC/B,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,UAAU,CAAC,gBAAgB,4BAA4B,CAAC;YACpF,CAAC,CAAC,EACV,EAAE,CAAC,CAAC;IAER,MAAM,UAAU,CAAC,QAAQ,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;IAEpC,IAAI,WAAW;QACX,OAAO,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC,CAAC;IAElG,OAAO,UAAU,CAAC,kBAAkB,CAAC;AACzC,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,OAAe,EAAE,SAA2E;IACpI,IAAI,kBAAkB,GAAyB,SAAS,CAAC;IAEzD,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE;QAC1E,OAAO,SAAS,CAAC;IAErB,KAAK,MAAM,gBAAgB,IAAI,SAAS,EAAE,CAAC;QACvC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,GAAG,gBAAgB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAChF,MAAM,iBAAiB,GAAG,SAAS,IAAI,EAAE,CAAC;YAC1C,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YAEhE,IAAI,kBAAkB,IAAI,IAAI;gBAC1B,kBAAkB,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAC,aAAa,EAAE,IAAI,EAAC,CAAC,CAAC;qBAClE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;qBAC/B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAElC,KAAK,MAAM,iBAAiB,IAAI,kBAAkB,EAAE,CAAC;gBACjD,IAAI,iBAAiB,CAAC,UAAU,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;oBACnG,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;oBAChG,IAAI,cAAc,CAAC,UAAU,CAAC;wBAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;gBACrD,CAAC;YACL,CAAC;YAED,SAAS;QACb,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QACtD,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;YAC7B,OAAO,QAAQ,CAAC;IACxB,CAAC;IAED,OAAO,SAAS,CAAC;AACrB,CAAC"}
+{"version":3,"file":"resolveModelFile.js","sourceRoot":"","sources":["../../src/utils/resolveModelFile.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,kBAAkB,EAAC,MAAM,cAAc,CAAC;AAChD,OAAO,EAAC,eAAe,EAAC,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAC,qBAAqB,EAAC,MAAM,wCAAwC,CAAC;AAC7E,OAAO,EAAC,uBAAuB,EAAC,MAAM,8BAA8B,CAAC;AAGrE,OAAO,EAAC,qBAAqB,EAAC,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAC,qBAAqB,EAAC,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAC,cAAc,EAAC,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAsFrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAClC,SAAiB,EACjB,kBAAqD;IAErD,MAAM,EACF,SAAS,EACT,QAAQ,GAAG,MAAM,EACjB,MAAM,GAAG,KAAK,EACd,QAAQ,EACR,OAAO,EACP,GAAG,GAAG,IAAI,EACV,UAAU,EACV,sBAAsB,GAAG,IAAI,EAC7B,QAAQ,GAAG,CAAC,EACZ,MAAM,EACN,SAAS,EACT,MAAM,EACT,GAAG,OAAO,kBAAkB,KAAK,QAAQ;QACtC,CAAC,CAAC,EAAC,SAAS,EAAE,kBAAkB,EAAC;QACjC,CAAC,CAAC,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;IAEjC,MAAM,iBAAiB,GAAG,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,WAAW,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7C,IAAI,cAAc,GAAG,MAAM,IAAI,KAAK,CAAC;IAErC,IAAI,QAAQ,KAAK,KAAK;QAClB,cAAc,GAAG,KAAK,CAAC;IAE3B,MAAM,wBAAwB,GAAG,uBAAuB,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAE1F,IAAI,wBAAwB,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3C,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAEpE,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC;YACrC,OAAO,gBAAgB,CAAC;QAE5B,MAAM,IAAI,KAAK,CAAC,2BAA2B,gBAAgB,GAAG,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,iBAAiB,GAAa,QAAQ,IAAI,IAAI;QAChD,CAAC,CAAC,CAAC,QAAQ,CAAC;QACZ,CAAC,CAAC,EAAE,CAAC;IAET,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,wBAAwB,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC5E,IAAI,wBAAwB,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU;YACtD,iBAAiB,CAAC,IAAI,CAAC,wBAAwB,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;;YAExE,OAAO,CAAC,iBAAiB,EAAE,wBAAwB,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;IAC7F,CAAC;SAAM,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,wBAAwB,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACnF,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QAC9E,IAAI,iBAAiB,IAAI,IAAI,IAAI,iBAAiB,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAChE,IAAI,iBAAiB,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU;gBAC/C,iBAAiB,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;;gBAEjE,OAAO,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;QACtF,CAAC;IACL,CAAC;IAED,MAAM,qBAAqB,GAAG,MAAM,4BAA4B,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;IAEvG,IAAI,qBAAqB,IAAI,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACnD,MAAM,YAAY,GAAG,qBAAqB,CAAC,qBAAqB,CAAC,CAAC;QAClE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,KAAK,qBAAqB;YACtE,OAAO,qBAAqB,CAAC;QAEjC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC;gBACvC,OAAO,YAAY,CAAC,CAAC,CAAE,CAAC;iBACvB,IAAI,QAAQ,KAAK,KAAK;gBACvB,MAAM,IAAI,KAAK,CAAC,0CAA0C,YAAY,CAAC,CAAC,CAAC,qCAAqC,CAAC,CAAC;QACxH,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;QACrB,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC,CAAE,CAAC,4BAA4B,CAAC,CAAC;QAEhI,MAAM,IAAI,KAAK,CAAC,4BAA4B,SAAS,SAAS,iBAAiB,4BAA4B,CAAC,CAAC;IACjH,CAAC;IAED,IAAI,MAAM,EAAE,OAAO;QACf,MAAM,MAAM,CAAC,MAAM,CAAC;IAExB,MAAM,UAAU,GAAG,MAAM,qBAAqB,CAAC;QAC3C,QAAQ,EAAE,wBAAwB,CAAC,IAAI,KAAK,KAAK;YAC7C,CAAC,CAAC,wBAAwB,CAAC,GAAG;YAC9B,CAAC,CAAC,wBAAwB,CAAC,GAAG;QAClC,OAAO,EAAE,iBAAiB;QAC1B,OAAO;QACP,eAAe,EAAE,WAAW;QAC5B,sBAAsB;QACtB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,QAAQ,IAAI,SAAS;QAC/B,iBAAiB,EAAE,QAAQ;QAC3B,UAAU;QACV,MAAM;QACN,SAAS;KACZ,CAAC,CAAC;IAEH,IAAI,qBAAqB,IAAI,IAAI,IAAI,UAAU,CAAC,UAAU,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACrH,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAEvD,IAAI,UAAU,CAAC,SAAS,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YAC1C,MAAM,UAAU,CAAC,MAAM,CAAC,EAAC,cAAc,EAAE,KAAK,EAAC,CAAC,CAAC;YACjD,OAAO,qBAAqB,CAAC;QACjC,CAAC;IACL,CAAC;IAED,IAAI,WAAW;QACX,OAAO,CAAC,IAAI,CAAC,kBAAkB,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,GAC3E,UAAU,CAAC,gBAAgB,IAAI,IAAI;YAC/B,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,UAAU,CAAC,gBAAgB,4BAA4B,CAAC;YACpF,CAAC,CAAC,EACV,EAAE,CAAC,CAAC;IAER,MAAM,UAAU,CAAC,QAAQ,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;IAEpC,IAAI,WAAW;QACX,OAAO,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC,CAAC;IAElG,OAAO,UAAU,CAAC,kBAAkB,CAAC;AACzC,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,OAAe,EAAE,SAA2E;IACpI,IAAI,kBAAkB,GAAyB,SAAS,CAAC;IAEzD,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE;QAC1E,OAAO,SAAS,CAAC;IAErB,KAAK,MAAM,gBAAgB,IAAI,SAAS,EAAE,CAAC;QACvC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,GAAG,gBAAgB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAChF,MAAM,iBAAiB,GAAG,SAAS,IAAI,EAAE,CAAC;YAC1C,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YAEhE,IAAI,kBAAkB,IAAI,IAAI;gBAC1B,kBAAkB,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAC,aAAa,EAAE,IAAI,EAAC,CAAC,CAAC;qBAClE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;qBAC/B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAElC,KAAK,MAAM,iBAAiB,IAAI,kBAAkB,EAAE,CAAC;gBACjD,IAAI,iBAAiB,CAAC,UAAU,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;oBACnG,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;oBAChG,IAAI,cAAc,CAAC,UAAU,CAAC;wBAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;gBACrD,CAAC;YACL,CAAC;YAED,SAAS;QACb,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QACtD,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;YAC7B,OAAO,QAAQ,CAAC;IACxB,CAAC;IAED,OAAO,SAAS,CAAC;AACrB,CAAC"}

package/dist/utils/transformPromisable.d.ts

@@ -37,4 +37,18 @@ export declare function promisableLoop<R>({ condition, callback, afterthought, r
     /** The value to return when the loop is done */
     returnValue: () => Promisable<R>;
 }): Promisable<R>;
+/**
+ * Calls the given getters in order, and when a promise is encountered, waits for it to resolve before continuing.
+ * The result is transformed using the transformer function.
+ *
+ * This is used as a performance optimization to avoid adding many microtasks to the event loop,
+ * which makes reading from buffers significantly faster.
+ * @param getters - An array of functions that return values or promises
+ * @param transformer - The transformer function that takes the promisable values and transforms them into the result of this function
+ */
+export declare function transformPromisablesInOrder<const T extends (() => Promisable<any>)[], const R = {
+    readonly [Index in keyof T]: Awaited<ReturnType<T[Index]>>;
+}>(getters: T, transformer?: (values: {
+    readonly [Index in keyof T]: Awaited<ReturnType<T[Index]>>;
+}) => Promisable<R>): Promisable<R>;
 export type Promisable<T> = T | Promise<T>;

package/dist/utils/transformPromisable.js

@@ -60,4 +60,36 @@ export function promisableLoop({ condition, callback, afterthought = () => void
     }
     return iterate();
 }
+/**
+ * Calls the given getters in order, and when a promise is encountered, waits for it to resolve before continuing.
+ * The result is transformed using the transformer function.
+ *
+ * This is used as a performance optimization to avoid adding many microtasks to the event loop,
+ * which makes reading from buffers significantly faster.
+ * @param getters - An array of functions that return values or promises
+ * @param transformer - The transformer function that takes the promisable values and transforms them into the result of this function
+ */
+export function transformPromisablesInOrder(getters, transformer = ((values) => values)) {
+    let i = 0;
+    const res = [];
+    let skipPushingValue = true;
+    function iterate(currentValue) {
+        if (skipPushingValue)
+            skipPushingValue = false;
+        else
+            res.push(currentValue);
+        while (i < getters.length) {
+            const getter = getters[i];
+            if (getter == null)
+                break;
+            i++;
+            const value = getter();
+            if (value instanceof Promise)
+                return value.then(iterate);
+            res.push(value);
+        }
+        return transformer(res);
+    }
+    return iterate(undefined);
+}
 //# sourceMappingURL=transformPromisable.js.map

package/dist/utils/transformPromisable.js.map

@@ -1 +1 @@
-{"version":3,"file":"transformPromisable.js","sourceRoot":"","sources":["../../src/utils/transformPromisable.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAO,KAAoB,EAAE,WAAwC;IACpG,IAAI,KAAK,YAAY,OAAO;QACxB,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEnC,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAChC,MAA0D,EAC1D,WAA8E;IAE9E,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,YAAY,OAAO,CAAC;QAChD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEjD,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAI,EAC9B,SAAS,EACT,QAAQ,EACR,YAAY,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,EAC3B,WAAW,EAad;IACG,SAAS,OAAO;QACZ,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,cAAc,GAAG,SAAS,EAAE,CAAC;YAEnC,IAAI,cAAc,YAAY,OAAO;gBACjC,OAAO,cAAc;qBAChB,IAAI,CAAC,CAAC,cAAc,EAAiB,EAAE;oBACpC,IAAI,cAAc,EAAE,CAAC;wBACjB,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;wBACzB,IAAI,KAAK,YAAY,OAAO;4BACxB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;wBAE1E,OAAO,mBAAmB,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,CAAC;oBACxD,CAAC;oBAED,OAAO,WAAW,EAAE,CAAC;gBACzB,CAAC,CAAC,CAAC;YAEX,IAAI,cAAc,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;gBACzB,IAAI,KAAK,YAAY,OAAO;oBACxB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;gBAE1E,MAAM,iBAAiB,GAAG,YAAY,EAAE,CAAC;gBACzC,IAAI,iBAAiB,YAAY,OAAO;oBACpC,OAAO,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAE3C,SAAS;YACb,CAAC;YAED,OAAO,WAAW,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;IAED,OAAO,OAAO,EAAE,CAAC;AACrB,CAAC"}
+{"version":3,"file":"transformPromisable.js","sourceRoot":"","sources":["../../src/utils/transformPromisable.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAO,KAAoB,EAAE,WAAwC;IACpG,IAAI,KAAK,YAAY,OAAO;QACxB,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEnC,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAChC,MAA0D,EAC1D,WAA8E;IAE9E,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,YAAY,OAAO,CAAC;QAChD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEjD,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAI,EAC9B,SAAS,EACT,QAAQ,EACR,YAAY,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,EAC3B,WAAW,EAad;IACG,SAAS,OAAO;QACZ,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,cAAc,GAAG,SAAS,EAAE,CAAC;YAEnC,IAAI,cAAc,YAAY,OAAO;gBACjC,OAAO,cAAc;qBAChB,IAAI,CAAC,CAAC,cAAc,EAAiB,EAAE;oBACpC,IAAI,cAAc,EAAE,CAAC;wBACjB,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;wBACzB,IAAI,KAAK,YAAY,OAAO;4BACxB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;wBAE1E,OAAO,mBAAmB,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,CAAC;oBACxD,CAAC;oBAED,OAAO,WAAW,EAAE,CAAC;gBACzB,CAAC,CAAC,CAAC;YAEX,IAAI,cAAc,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;gBACzB,IAAI,KAAK,YAAY,OAAO;oBACxB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;gBAE1E,MAAM,iBAAiB,GAAG,YAAY,EAAE,CAAC;gBACzC,IAAI,iBAAiB,YAAY,OAAO;oBACpC,OAAO,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAE3C,SAAS;YACb,CAAC;YAED,OAAO,WAAW,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;IAED,OAAO,OAAO,EAAE,CAAC;AACrB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,2BAA2B,CAIvC,OAAU,EACV,cAEqB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAW,CAAC;IAE9C,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAE5B,SAAS,OAAO,CAAC,YAAiB;QAC9B,IAAI,gBAAgB;YAChB,gBAAgB,GAAG,KAAK,CAAC;;YAEzB,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE3B,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,MAAM,IAAI,IAAI;gBACd,MAAM;YAEV,CAAC,EAAE,CAAC;YAEJ,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;YACvB,IAAI,KAAK,YAAY,OAAO;gBACxB,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE/B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;QAED,OAAO,WAAW,CAAC,GAA2D,CAAC,CAAC;IACpF,CAAC;IAED,OAAO,OAAO,CAAC,SAAS,CAAC,CAAC;AAC9B,CAAC"}

package/llama/CMakeLists.txt

@@ -1,9 +1,17 @@
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.19)
 
 if (NLC_CURRENT_PLATFORM STREQUAL "win-x64" OR NLC_CURRENT_PLATFORM STREQUAL "win-arm64")
     set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 endif()
 
+if (NLC_CURRENT_PLATFORM STREQUAL "win-x64")
+    if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+        set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebugDLL" CACHE STRING "" FORCE)
+    else()
+        set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDLL" CACHE STRING "" FORCE)
+    endif()
+endif()
+
 if (NLC_TARGET_PLATFORM STREQUAL "win-arm64" AND (CMAKE_GENERATOR STREQUAL "Ninja" OR CMAKE_GENERATOR STREQUAL "Ninja Multi-Config") AND NOT MINGW)
     if(NLC_CURRENT_PLATFORM STREQUAL "win-x64")
         include("./profiles/llvm.win32.host-x64.target-arm64.cmake")
@@ -70,6 +78,9 @@ add_subdirectory("llama.cpp")
 include_directories("llama.cpp")
 include_directories("./llama.cpp/common")
 
+# This is needed to use methods in "llama-grammar.h" and "unicode.h"
+target_include_directories(llama PUBLIC "./llama.cpp/src")
+
 unset(GPU_INFO_HEADERS)
 unset(GPU_INFO_SOURCES)
 unset(GPU_INFO_EXTRA_LIBS)

package/llama/addon/AddonContext.cpp

@@ -2,7 +2,6 @@
 #include <algorithm>
 #include <cmath>
 #include "common/common.h"
-#include "llama-grammar.h"
 #include "llama.h"
 
 #include "addonGlobals.h"
@@ -583,7 +582,7 @@ Napi::Value AddonContext::DisposeSequence(const Napi::CallbackInfo& info) {
 
     int32_t sequenceId = info[0].As<Napi::Number>().Int32Value();
 
-    bool result = llama_kv_cache_seq_rm(ctx, sequenceId, -1, -1);
+    bool result = llama_kv_self_seq_rm(ctx, sequenceId, -1, -1);
 
     if (!result) {
         Napi::Error::New(info.Env(), "Failed to dispose sequence").ThrowAsJavaScriptException();
@@ -602,7 +601,7 @@ Napi::Value AddonContext::RemoveTokenCellsFromSequence(const Napi::CallbackInfo&
     int32_t startPos = info[1].As<Napi::Number>().Int32Value();
     int32_t endPos = info[2].As<Napi::Number>().Int32Value();
 
-    bool result = llama_kv_cache_seq_rm(ctx, sequenceId, startPos, endPos);
+    bool result = llama_kv_self_seq_rm(ctx, sequenceId, startPos, endPos);
 
     return Napi::Boolean::New(info.Env(), result);
 }
@@ -617,7 +616,7 @@ Napi::Value AddonContext::ShiftSequenceTokenCells(const Napi::CallbackInfo& info
     int32_t endPos = info[2].As<Napi::Number>().Int32Value();
     int32_t shiftDelta = info[3].As<Napi::Number>().Int32Value();
 
-    llama_kv_cache_seq_add(ctx, sequenceId, startPos, endPos, shiftDelta);
+    llama_kv_self_seq_add(ctx, sequenceId, startPos, endPos, shiftDelta);
 
     return info.Env().Undefined();
 }
@@ -639,6 +638,7 @@ Napi::Value AddonContext::GetEmbedding(const Napi::CallbackInfo& info) {
     }
 
     int32_t inputTokensLength = info[0].As<Napi::Number>().Int32Value();
+    int32_t maxVectorSize = (info.Length() > 1 && info[1].IsNumber()) ? info[1].As<Napi::Number>().Int32Value() : 0;
 
     if (inputTokensLength <= 0) {
         Napi::Error::New(info.Env(), "Invalid input tokens length").ThrowAsJavaScriptException();
@@ -650,15 +650,16 @@ Napi::Value AddonContext::GetEmbedding(const Napi::CallbackInfo& info) {
     const auto* embeddings = pooling_type == LLAMA_POOLING_TYPE_NONE ? NULL : llama_get_embeddings_seq(ctx, 0);
     if (embeddings == NULL) {
         embeddings = llama_get_embeddings_ith(ctx, inputTokensLength - 1);
+    }
 
-        if (embeddings == NULL) {
-            Napi::Error::New(info.Env(), std::string("Failed to get embeddings for token ") + std::to_string(inputTokensLength - 1)).ThrowAsJavaScriptException();
-            return info.Env().Undefined();
-        }
+    if (embeddings == NULL) {
+        Napi::Error::New(info.Env(), std::string("Failed to get embeddings for token ") + std::to_string(inputTokensLength - 1)).ThrowAsJavaScriptException();
+        return info.Env().Undefined();
     }
 
-    Napi::Float64Array result = Napi::Float64Array::New(info.Env(), n_embd);
-    for (size_t i = 0; i < n_embd; ++i) {
+    size_t resultSize = maxVectorSize == 0 ? n_embd : std::min(n_embd, maxVectorSize);
+    Napi::Float64Array result = Napi::Float64Array::New(info.Env(), resultSize);
+    for (size_t i = 0; i < resultSize; i++) {
         result[i] = embeddings[i];
     }
 
@@ -701,6 +702,143 @@ Napi::Value AddonContext::SetThreads(const Napi::CallbackInfo& info) {
     return info.Env().Undefined();
 }
 
+class AddonContextSaveSequenceStateToFileWorker : public Napi::AsyncWorker {
+    public:
+        AddonContext* context;
+        std::string filepath;
+        llama_seq_id sequenceId;
+        std::vector<llama_token> tokens;
+        size_t savedFileSize = 0;
+
+        AddonContextSaveSequenceStateToFileWorker(const Napi::CallbackInfo& info, AddonContext* context)
+            : Napi::AsyncWorker(info.Env(), "AddonContextSaveSequenceStateToFileWorker"),
+              context(context),
+              deferred(Napi::Promise::Deferred::New(info.Env())) {
+            context->Ref();
+
+            filepath = info[0].As<Napi::String>().Utf8Value();
+            sequenceId = info[1].As<Napi::Number>().Int32Value();
+            Napi::Uint32Array inputTokens = info[2].As<Napi::Uint32Array>();
+
+            tokens.resize(inputTokens.ElementLength());
+            for (size_t i = 0; i < tokens.size(); i++) {
+                tokens[i] = inputTokens[i];
+            }
+        }
+        ~AddonContextSaveSequenceStateToFileWorker() {
+            context->Unref();
+        }
+
+        Napi::Promise GetPromise() {
+            return deferred.Promise();
+        }
+
+    protected:
+        Napi::Promise::Deferred deferred;
+
+        void Execute() {
+            try {
+                savedFileSize = llama_state_seq_save_file(context->ctx, filepath.c_str(), sequenceId, tokens.data(), tokens.size());
+                if (savedFileSize == 0) {
+                    SetError("Failed to save state to file");
+                    return;
+                }
+            } catch (const std::exception& e) {
+                SetError(e.what());
+            } catch(...) {
+                SetError("Unknown error when calling \"llama_state_seq_save_file\"");
+            }
+        }
+        void OnOK() {
+            deferred.Resolve(Napi::Number::New(Env(), savedFileSize));
+        }
+        void OnError(const Napi::Error& err) {
+            deferred.Reject(err.Value());
+        }
+};
+Napi::Value AddonContext::SaveSequenceStateToFile(const Napi::CallbackInfo& info) {
+    if (disposed) {
+        Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
+
+    AddonContextSaveSequenceStateToFileWorker* worker = new AddonContextSaveSequenceStateToFileWorker(info, this);
+    worker->Queue();
+    return worker->GetPromise();
+}
+
+class AddonContextLoadSequenceStateFromFileWorker : public Napi::AsyncWorker {
+    public:
+        AddonContext* context;
+        std::string filepath;
+        llama_seq_id sequenceId;
+        size_t maxContextSize;
+        std::vector<llama_token> tokens;
+
+        AddonContextLoadSequenceStateFromFileWorker(const Napi::CallbackInfo& info, AddonContext* context)
+            : Napi::AsyncWorker(info.Env(), "AddonContextLoadSequenceStateFromFileWorker"),
+              context(context),
+              deferred(Napi::Promise::Deferred::New(info.Env())) {
+            context->Ref();
+
+            filepath = info[0].As<Napi::String>().Utf8Value();
+            sequenceId = info[1].As<Napi::Number>().Int32Value();
+            maxContextSize = info[2].As<Napi::Number>().Uint32Value();
+
+            tokens.resize(maxContextSize);
+        }
+        ~AddonContextLoadSequenceStateFromFileWorker() {
+            context->Unref();
+        }
+
+        Napi::Promise GetPromise() {
+            return deferred.Promise();
+        }
+
+    protected:
+        Napi::Promise::Deferred deferred;
+
+        void Execute() {
+            try {
+                size_t tokenCount = 0;
+                const size_t fileSize = llama_state_seq_load_file(context->ctx, filepath.c_str(), sequenceId, tokens.data(), tokens.size(), &tokenCount);
+                if (fileSize == 0) {
+                    SetError("Failed to load state from file. Current context sequence size may be smaller that the state of the file");
+                    return;
+                }
+
+                tokens.resize(tokenCount);
+            } catch (const std::exception& e) {
+                SetError(e.what());
+            } catch(...) {
+                SetError("Unknown error when calling \"llama_state_seq_load_file\"");
+            }
+        }
+        void OnOK() {
+            size_t tokenCount = tokens.size();
+            Napi::Uint32Array result = Napi::Uint32Array::New(Env(), tokenCount);
+
+            for (size_t i = 0; i < tokenCount; i++) {
+                result[i] = tokens[i];
+            }
+
+            deferred.Resolve(result);
+        }
+        void OnError(const Napi::Error& err) {
+            deferred.Reject(err.Value());
+        }
+};
+Napi::Value AddonContext::LoadSequenceStateFromFile(const Napi::CallbackInfo& info) {
+    if (disposed) {
+        Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
+
+    AddonContextLoadSequenceStateFromFileWorker* worker = new AddonContextLoadSequenceStateFromFileWorker(info, this);
+    worker->Queue();
+    return worker->GetPromise();
+}
+
 Napi::Value AddonContext::PrintTimings(const Napi::CallbackInfo& info) {
     llama_perf_context_print(ctx);
     llama_perf_context_reset(ctx);
@@ -796,6 +934,8 @@ void AddonContext::init(Napi::Object exports) {
                 InstanceMethod("setThreads", &AddonContext::SetThreads),
                 InstanceMethod("printTimings", &AddonContext::PrintTimings),
                 InstanceMethod("ensureDraftContextIsCompatibleForSpeculative", &AddonContext::EnsureDraftContextIsCompatibleForSpeculative),
+                InstanceMethod("saveSequenceStateToFile", &AddonContext::SaveSequenceStateToFile),
+                InstanceMethod("loadSequenceStateFromFile", &AddonContext::LoadSequenceStateFromFile),
                 InstanceMethod("setLora", &AddonContext::SetLora),
                 InstanceMethod("dispose", &AddonContext::Dispose),
             }

package/llama/addon/AddonContext.h

@@ -44,6 +44,9 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
         Napi::Value GetThreads(const Napi::CallbackInfo& info);
         Napi::Value SetThreads(const Napi::CallbackInfo& info);
 
+        Napi::Value SaveSequenceStateToFile(const Napi::CallbackInfo& info);
+        Napi::Value LoadSequenceStateFromFile(const Napi::CallbackInfo& info);
+
         Napi::Value PrintTimings(const Napi::CallbackInfo& info);
         Napi::Value EnsureDraftContextIsCompatibleForSpeculative(const Napi::CallbackInfo& info);
 

package/llama/addon/AddonSampler.cpp

@@ -1,6 +1,5 @@
 #include <cmath>
 #include "common/common.h"
-#include "llama-grammar.h"
 #include "llama.h"
 
 #include "AddonGrammarEvaluationState.h"

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b4753"
+    "release": "b5414"
 }

package/llama/gpuInfo/vulkan-gpu-info.cpp

@@ -66,7 +66,6 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, size_t* unifiedM
                         features2.pNext = &vk11Features;
 
                         vkGetPhysicalDeviceFeatures2(physicalDevice, &features2);
-                        VkPhysicalDeviceFeatures2 device_features2;
 
                         if (!vk11Features.storageBuffer16BitAccess) {
                             *checkSupported = false;

package/llama/grammars/README.md

@@ -1,6 +1,6 @@
 # GBNF Guide
 
-GBNF (GGML BNF) is a format for defining [formal grammars](https://en.wikipedia.org/wiki/Formal_grammar) to constrain model outputs in `llama.cpp`. For example, you can use it to force the model to generate valid JSON, or speak only in emojis. GBNF grammars are supported in various ways in `examples/main` and `examples/server`.
+GBNF (GGML BNF) is a format for defining [formal grammars](https://en.wikipedia.org/wiki/Formal_grammar) to constrain model outputs in `llama.cpp`. For example, you can use it to force the model to generate valid JSON, or speak only in emojis. GBNF grammars are supported in various ways in `tools/main` and `tools/server`.
 
 ## Background
 
@@ -110,21 +110,21 @@ While semantically correct, the syntax `x? x? x?.... x?` (with N repetitions) ma
 
 You can use GBNF grammars:
 
-- In [llama-server](../examples/server)'s completion endpoints, passed as the `grammar` body field
-- In [llama-cli](../examples/main), passed as the `--grammar` & `--grammar-file` flags
-- With [llama-gbnf-validator](../examples/gbnf-validator) tool, to test them against strings.
+- In [llama-server](../tools/server)'s completion endpoints, passed as the `grammar` body field
+- In [llama-cli](../tools/main), passed as the `--grammar` & `--grammar-file` flags
+- With [test-gbnf-validator](../tests/test-gbnf-validator.cpp), to test them against strings.
 
 ## JSON Schemas → GBNF
 
 `llama.cpp` supports converting a subset of https://json-schema.org/ to GBNF grammars:
 
-- In [llama-server](../examples/server):
+- In [llama-server](../tools/server):
     - For any completion endpoints, passed as the `json_schema` body field
     - For the `/chat/completions` endpoint, passed inside the `response_format` body field (e.g. `{"type", "json_object", "schema": {"items": {}}}` or `{ type: "json_schema", json_schema: {"schema": ...} }`)
-- In [llama-cli](../examples/main), passed as the `--json` / `-j` flag
+- In [llama-cli](../tools/main), passed as the `--json` / `-j` flag
 - To convert to a grammar ahead of time:
     - in CLI, with [examples/json_schema_to_grammar.py](../examples/json_schema_to_grammar.py)
-    - in JavaScript with [json-schema-to-grammar.mjs](../examples/server/public_legacy/json-schema-to-grammar.mjs) (this is used by the [server](../examples/server)'s Web UI)
+    - in JavaScript with [json-schema-to-grammar.mjs](../tools/server/public_legacy/json-schema-to-grammar.mjs) (this is used by the [server](../tools/server)'s Web UI)
 
 Take a look at [tests](../tests/test-json-schema-to-grammar.cpp) to see which features are likely supported (you'll also find usage examples in https://github.com/ggml-org/llama.cpp/pull/5978, https://github.com/ggml-org/llama.cpp/pull/6659 & https://github.com/ggml-org/llama.cpp/pull/6555).
 

package/llama/llama.cpp.info.json

@@ -1,4 +1,4 @@
 {
-    "tag": "b4753",
+    "tag": "b5414",
     "llamaCppGithubRepo": "ggml-org/llama.cpp"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-llama-cpp",
-  "version": "3.6.0",
+  "version": "3.8.0",
   "description": "Run AI models locally on your machine with node.js bindings for llama.cpp. Enforce a JSON schema on the model output on the generation level",
   "main": "./dist/index.js",
   "type": "module",
@@ -39,10 +39,10 @@
     }
   },
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=20.0.0"
   },
   "scripts": {
-    "prepare": "[ \"$CI\" = true ] || [ -d '.husky/_' ] || husky",
+    "prepare": "node --experimental-require-module -e \"process.env.CI !== 'true' && console.log(require('husky').default())\"",
     "postinstall": "node ./dist/cli/cli.js postinstall",
     "postversion": "vite-node scripts/postVersion.ts",
     "prebuild": "rimraf ./dist ./tsconfig.tsbuildinfo",
@@ -111,6 +111,8 @@
     "catai",
     "mistral",
     "deepseek",
+    "qwen",
+    "qwq",
     "typescript",
     "lora",
     "batching",
@@ -131,78 +133,78 @@
   },
   "homepage": "https://node-llama-cpp.withcat.ai",
   "devDependencies": {
-    "@commitlint/cli": "^19.6.1",
-    "@commitlint/config-conventional": "^19.6.0",
-    "@eslint/compat": "^1.2.5",
-    "@fontsource/inter": "^5.1.1",
-    "@nolebase/vitepress-plugin-git-changelog": "^2.12.1",
-    "@nolebase/vitepress-plugin-og-image": "^2.12.1",
+    "@commitlint/cli": "^19.8.1",
+    "@commitlint/config-conventional": "^19.8.1",
+    "@eslint/compat": "^1.2.9",
+    "@fontsource/inter": "^5.2.5",
+    "@nolebase/vitepress-plugin-git-changelog": "^2.17.0",
+    "@nolebase/vitepress-plugin-og-image": "^2.17.0",
     "@resvg/resvg-js": "^2.6.2",
-    "@semantic-release/exec": "^6.0.3",
-    "@semantic-release/github": "11.0.1",
+    "@semantic-release/exec": "^7.1.0",
+    "@semantic-release/github": "11.0.2",
     "@semantic-release/npm": "12.0.1",
-    "@shikijs/vitepress-twoslash": "^2.2.0",
-    "@stylistic/eslint-plugin": "^3.0.1",
+    "@shikijs/vitepress-twoslash": "^3.4.0",
+    "@stylistic/eslint-plugin": "^4.2.0",
     "@types/async-retry": "^1.4.9",
     "@types/bytes": "^3.1.5",
     "@types/cross-spawn": "^6.0.6",
     "@types/fs-extra": "^11.0.4",
-    "@types/node": "^22.12.0",
+    "@types/node": "^22.15.17",
     "@types/proper-lockfile": "^4.1.4",
-    "@types/semver": "^7.5.8",
+    "@types/semver": "^7.7.0",
     "@types/validate-npm-package-name": "^4.0.2",
     "@types/which": "^3.0.4",
     "@types/yargs": "^17.0.33",
-    "@vitest/coverage-v8": "^3.0.4",
-    "@vitest/ui": "^3.0.4",
-    "electron": "^34.0.2",
-    "eslint": "^9.19.0",
-    "eslint-import-resolver-typescript": "^3.7.0",
+    "@vitest/coverage-v8": "^3.1.3",
+    "@vitest/ui": "^3.1.3",
+    "electron": "^36.2.0",
+    "eslint": "^9.26.0",
+    "eslint-import-resolver-typescript": "^4.3.4",
     "eslint-plugin-import": "^2.31.0",
-    "eslint-plugin-jsdoc": "^50.6.3",
-    "eslint-plugin-n": "^17.15.1",
-    "feed": "^4.2.2",
+    "eslint-plugin-jsdoc": "^50.6.14",
+    "eslint-plugin-n": "^17.18.0",
+    "feed": "^5.0.0",
     "husky": "^9.1.7",
     "rehype": "^13.0.2",
     "rimraf": "^6.0.1",
-    "semantic-release": "^24.2.1",
-    "sharp": "^0.33.5",
+    "semantic-release": "^24.2.3",
+    "sharp": "^0.34.1",
     "tslib": "^2.8.1",
-    "typedoc": "^0.27.6",
-    "typedoc-plugin-markdown": "^4.4.1",
-    "typedoc-plugin-mdn-links": "^4.0.10",
+    "typedoc": "^0.28.4",
+    "typedoc-plugin-markdown": "^4.6.3",
+    "typedoc-plugin-mdn-links": "^5.0.2",
     "typedoc-vitepress-theme": "^1.1.2",
-    "typescript": "^5.7.3",
-    "typescript-eslint": "^8.22.0",
-    "vite-node": "^3.0.4",
+    "typescript": "^5.8.3",
+    "typescript-eslint": "^8.32.0",
+    "vite-node": "^3.1.3",
     "vitepress": "^1.6.3",
-    "vitest": "^3.0.4",
-    "zx": "^8.3.1"
+    "vitest": "^3.1.3",
+    "zx": "^8.5.4"
   },
   "dependencies": {
-    "@huggingface/jinja": "^0.3.3",
+    "@huggingface/jinja": "^0.5.0",
     "async-retry": "^1.3.3",
     "bytes": "^3.1.2",
     "chalk": "^5.4.1",
     "chmodrp": "^1.0.2",
-    "cmake-js": "^7.3.0",
+    "cmake-js": "^7.3.1",
     "cross-env": "^7.0.3",
     "cross-spawn": "^7.0.6",
     "env-var": "^7.5.0",
     "filenamify": "^6.0.0",
     "fs-extra": "^11.3.0",
-    "ignore": "^7.0.0",
+    "ignore": "^7.0.4",
     "ipull": "^3.9.2",
     "is-unicode-supported": "^2.1.0",
     "lifecycle-utils": "^2.0.0",
     "log-symbols": "^7.0.0",
-    "nanoid": "^5.0.9",
-    "node-addon-api": "^8.3.0",
-    "octokit": "^4.1.0",
-    "ora": "^8.1.1",
+    "nanoid": "^5.1.5",
+    "node-addon-api": "^8.3.1",
+    "octokit": "^4.1.3",
+    "ora": "^8.2.0",
     "pretty-ms": "^9.2.0",
     "proper-lockfile": "^4.1.2",
-    "semver": "^7.7.0",
+    "semver": "^7.7.1",
     "simple-git": "^3.27.0",
     "slice-ansi": "^7.1.0",
     "stdout-update": "^4.0.1",
@@ -220,16 +222,16 @@
     }
   },
   "optionalDependencies": {
-    "@node-llama-cpp/linux-arm64": "3.6.0",
-    "@node-llama-cpp/linux-armv7l": "3.6.0",
-    "@node-llama-cpp/linux-x64": "3.6.0",
-    "@node-llama-cpp/linux-x64-cuda": "3.6.0",
-    "@node-llama-cpp/linux-x64-vulkan": "3.6.0",
-    "@node-llama-cpp/mac-arm64-metal": "3.6.0",
-    "@node-llama-cpp/mac-x64": "3.6.0",
-    "@node-llama-cpp/win-arm64": "3.6.0",
-    "@node-llama-cpp/win-x64": "3.6.0",
-    "@node-llama-cpp/win-x64-cuda": "3.6.0",
-    "@node-llama-cpp/win-x64-vulkan": "3.6.0"
+    "@node-llama-cpp/linux-arm64": "3.8.0",
+    "@node-llama-cpp/linux-armv7l": "3.8.0",
+    "@node-llama-cpp/linux-x64": "3.8.0",
+    "@node-llama-cpp/linux-x64-cuda": "3.8.0",
+    "@node-llama-cpp/linux-x64-vulkan": "3.8.0",
+    "@node-llama-cpp/mac-arm64-metal": "3.8.0",
+    "@node-llama-cpp/mac-x64": "3.8.0",
+    "@node-llama-cpp/win-arm64": "3.8.0",
+    "@node-llama-cpp/win-x64": "3.8.0",
+    "@node-llama-cpp/win-x64-cuda": "3.8.0",
+    "@node-llama-cpp/win-x64-vulkan": "3.8.0"
   }
 }