node-llama-cpp 0.0.1 → 1.1.1

package/dist/utils/getBin.js

@@ -0,0 +1,70 @@
+import { createRequire } from "module";
+import * as console from "console";
+import path from "path";
+import process from "process";
+import fs from "fs-extra";
+import { defaultLlamaCppGitHubRepo, defaultLlamaCppRelease, defaultSkipDownload, llamaBinsDirectory } from "../config.js";
+import { DownloadLlamaCppCommand } from "../cli/commands/DownloadCommand.js";
+import { getUsedBinFlag } from "./usedBinFlag.js";
+import { getCompiledLlamaCppBinaryPath } from "./compileLLamaCpp.js";
+const require = createRequire(import.meta.url);
+export async function getPrebuildBinPath() {
+    const majorNodeVersion = parseInt(process.version.slice("v".length));
+    const supportedVersions = [majorNodeVersion, majorNodeVersion - 1];
+    function createPath(platform, arch, nodeVersion) {
+        return path.join(llamaBinsDirectory, `${platform}-${arch}-${nodeVersion}.node`);
+    }
+    async function resolvePath(platform, arch, nodeVersions) {
+        for (const nodeVersion of nodeVersions) {
+            const binPath = createPath(platform, arch, nodeVersion);
+            if (await fs.exists(binPath))
+                return binPath;
+        }
+        return null;
+    }
+    async function getPath() {
+        switch (process.platform) {
+            case "win32":
+            case "cygwin":
+                return resolvePath("win", process.arch, supportedVersions);
+            case "linux":
+            case "android":
+                return resolvePath("linux", process.arch, supportedVersions);
+            case "darwin":
+                return resolvePath("mac", process.arch, supportedVersions);
+        }
+        return null;
+    }
+    return await getPath();
+}
+export async function loadBin() {
+    const usedBingFlag = await getUsedBinFlag();
+    if (usedBingFlag === "prebuiltBinaries") {
+        const prebuildBinPath = await getPrebuildBinPath();
+        if (prebuildBinPath == null) {
+            console.warn("Prebuild binaries not found, falling back to to locally built binaries");
+        }
+        else {
+            return require(prebuildBinPath);
+        }
+    }
+    const modulePath = await getCompiledLlamaCppBinaryPath();
+    if (modulePath == null) {
+        if (defaultSkipDownload) {
+            throw new Error("No prebuild binaries found and NODE_LLAMA_CPP_SKIP_DOWNLOAD env var is set to true");
+        }
+        else {
+            await DownloadLlamaCppCommand({
+                repo: defaultLlamaCppGitHubRepo,
+                release: defaultLlamaCppRelease
+            });
+            const modulePath = await getCompiledLlamaCppBinaryPath();
+            if (modulePath == null) {
+                throw new Error("Failed to download and compile llama.cpp");
+            }
+            return require(modulePath);
+        }
+    }
+    return require(modulePath);
+}
+//# sourceMappingURL=getBin.js.map

package/dist/utils/getBin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getBin.js","sourceRoot":"","sources":["../../src/utils/getBin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,yBAAyB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,kBAAkB,EAAC,MAAM,cAAc,CAAC;AACxH,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,6BAA6B,EAAC,MAAM,sBAAsB,CAAC;AAEnE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACpC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACrE,MAAM,iBAAiB,GAAG,CAAC,gBAAgB,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAEnE,SAAS,UAAU,CAAC,QAAgB,EAAE,IAAY,EAAE,WAAmB;QACnE,OAAO,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,QAAQ,IAAI,IAAI,IAAI,WAAW,OAAO,CAAC,CAAC;IACpF,CAAC;IAED,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,IAAY,EAAE,YAAsB;QAC7E,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACpC,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;YAExD,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;gBACxB,OAAO,OAAO,CAAC;SACtB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,UAAU,OAAO;QAClB,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,OAAO,CAAC;YACb,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAE/D,KAAK,OAAO,CAAC;YACb,KAAK,SAAS;gBACV,OAAO,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAEjE,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;SAClE;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IACzB,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAE5C,IAAI,YAAY,KAAK,kBAAkB,EAAE;QACrC,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAEnD,IAAI,eAAe,IAAI,IAAI,EAAE;YACzB,OAAO,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;SAC1F;aAAM;YACH,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;SACnC;KACJ;IAED,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;IAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;QACpB,IAAI,mBAAmB,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;SACzG;aAAM;YACH,MAAM,uBAAuB,CAAC;gBAC1B,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,sBAAsB;aAClC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;YAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;aAC/D;YAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;SAC9B;KACJ;IAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;AAC/B,CAAC"}

package/dist/utils/spawnCommand.d.ts

@@ -0,0 +1 @@
+export declare function spawnCommand(command: string, args: string[], cwd: string): Promise<void>;

package/dist/utils/spawnCommand.js

@@ -0,0 +1,39 @@
+import spawn from "cross-spawn";
+export function spawnCommand(command, args, cwd) {
+    function getCommandString() {
+        let res = command;
+        for (const arg of args) {
+            if (arg.includes(" ")) {
+                res += ` "${arg.split('"').join('\\"')}"`;
+            }
+            else {
+                res += ` ${arg}`;
+            }
+        }
+        return res;
+    }
+    return new Promise((resolve, reject) => {
+        const child = spawn(command, args, {
+            stdio: "inherit",
+            cwd,
+            env: process.env,
+            detached: false,
+            windowsHide: true
+        });
+        child.on("exit", (code) => {
+            if (code == 0)
+                resolve();
+            else
+                reject(new Error(`Command ${getCommandString()} exited with code ${code}`));
+        });
+        child.on("error", reject);
+        child.on("disconnect", () => reject(new Error(`Command ${getCommandString()} disconnected`)));
+        child.on("close", code => {
+            if (code == 0)
+                resolve();
+            else
+                reject(new Error(`Command ${getCommandString()} closed with code ${code}`));
+        });
+    });
+}
+//# sourceMappingURL=spawnCommand.js.map

package/dist/utils/spawnCommand.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"spawnCommand.js","sourceRoot":"","sources":["../../src/utils/spawnCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAEhC,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,IAAc,EAAE,GAAW;IACrE,SAAS,gBAAgB;QACrB,IAAI,GAAG,GAAG,OAAO,CAAC;QAElB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACpB,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACnB,GAAG,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;aAC7C;iBAAM;gBACH,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;aACpB;SACJ;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAED,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACzC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE;YAC/B,KAAK,EAAE,SAAS;YAChB,GAAG;YACH,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SACpB,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACtB,IAAI,IAAI,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC;;gBAEV,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;QAC9F,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE;YACrB,IAAI,IAAI,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC;;gBAEV,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/utils/usedBinFlag.d.ts

@@ -0,0 +1,6 @@
+type UsedBinFlagFile = {
+    use: "prebuiltBinaries" | "localBuildFromSource";
+};
+export declare function getUsedBinFlag(): Promise<"prebuiltBinaries" | "localBuildFromSource">;
+export declare function setUsedBinFlag(useFlag: UsedBinFlagFile["use"]): Promise<void>;
+export {};

package/dist/utils/usedBinFlag.js

@@ -0,0 +1,15 @@
+import fs from "fs-extra";
+import { usedBinFlagJsonPath } from "../config.js";
+export async function getUsedBinFlag() {
+    const usedBinFlagJson = await fs.readJson(usedBinFlagJsonPath);
+    return usedBinFlagJson.use;
+}
+export async function setUsedBinFlag(useFlag) {
+    const usedBinFlagJson = {
+        use: useFlag
+    };
+    await fs.writeJson(usedBinFlagJsonPath, usedBinFlagJson, {
+        spaces: 4
+    });
+}
+//# sourceMappingURL=usedBinFlag.js.map

package/dist/utils/usedBinFlag.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"usedBinFlag.js","sourceRoot":"","sources":["../../src/utils/usedBinFlag.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,mBAAmB,EAAC,MAAM,cAAc,CAAC;AAMjD,MAAM,CAAC,KAAK,UAAU,cAAc;IAChC,MAAM,eAAe,GAAoB,MAAM,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAEhF,OAAO,eAAe,CAAC,GAAG,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA+B;IAChE,MAAM,eAAe,GAAoB;QACrC,GAAG,EAAE,OAAO;KACf,CAAC;IAEF,MAAM,EAAE,CAAC,SAAS,CAAC,mBAAmB,EAAE,eAAe,EAAE;QACrD,MAAM,EAAE,CAAC;KACZ,CAAC,CAAC;AACP,CAAC"}

package/dist/utils/withLock.d.ts

@@ -0,0 +1 @@
+export declare function withLock<ReturnType>(scope: any, key: string, callback: () => Promise<ReturnType>): Promise<ReturnType>;

package/dist/utils/withLock.js

@@ -0,0 +1,19 @@
+const locks = new Map();
+export async function withLock(scope, key, callback) {
+    while (locks.get(scope)?.has(key)) {
+        await locks.get(scope)?.get(key);
+    }
+    const promise = callback();
+    if (!locks.has(scope))
+        locks.set(scope, new Map());
+    locks.get(scope).set(key, promise);
+    try {
+        return await promise;
+    }
+    finally {
+        locks.get(scope)?.delete(key);
+        if (locks.get(scope)?.size === 0)
+            locks.delete(scope);
+    }
+}
+//# sourceMappingURL=withLock.js.map

package/dist/utils/withLock.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"withLock.js","sourceRoot":"","sources":["../../src/utils/withLock.ts"],"names":[],"mappings":"AAAA,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkC,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAa,KAAU,EAAE,GAAW,EAAE,QAAmC;IACnG,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE;QAC/B,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;KACpC;IAED,MAAM,OAAO,GAAG,QAAQ,EAAE,CAAC;IAE3B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;QACjB,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;IAEhC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAEpC,IAAI;QACA,OAAO,MAAM,OAAO,CAAC;KACxB;YAAS;QACN,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;QAE9B,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC;YAC5B,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;KAC3B;AACL,CAAC"}

package/dist/utils/withOra.d.ts

@@ -0,0 +1,5 @@
+export default function withOra<T>(message: string | {
+    loading: string;
+    success?: string;
+    fail?: string;
+}, callback: () => Promise<T>): Promise<T>;

package/dist/utils/withOra.js

@@ -0,0 +1,21 @@
+import ora from "ora";
+export default async function withOra(message, callback) {
+    const spinner = ora(typeof message === "string" ? message : message.loading);
+    spinner.start();
+    try {
+        const res = await callback();
+        if (typeof message !== "string")
+            spinner.succeed(message.success);
+        else
+            spinner.succeed();
+        return res;
+    }
+    catch (er) {
+        if (typeof message !== "string")
+            spinner.fail(message.fail);
+        else
+            spinner.fail();
+        throw er;
+    }
+}
+//# sourceMappingURL=withOra.js.map

package/dist/utils/withOra.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"withOra.js","sourceRoot":"","sources":["../../src/utils/withOra.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,KAAK,CAAC;AAEtB,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,OAAO,CACjC,OAIC,EACD,QAA0B;IAE1B,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7E,OAAO,CAAC,KAAK,EAAE,CAAC;IAEhB,IAAI;QACA,MAAM,GAAG,GAAG,MAAM,QAAQ,EAAE,CAAC;QAE7B,IAAI,OAAO,OAAO,KAAK,QAAQ;YAC3B,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;;YAEjC,OAAO,CAAC,OAAO,EAAE,CAAC;QAEtB,OAAO,GAAG,CAAC;KACd;IAAC,OAAO,EAAE,EAAE;QACT,IAAI,OAAO,OAAO,KAAK,QAAQ;YAC3B,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;;YAE3B,OAAO,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,EAAE,CAAC;KACZ;AACL,CAAC"}

package/llama/.clang-format

@@ -0,0 +1,46 @@
+BasedOnStyle: Google
+IndentWidth: 2
+UseTab: Never
+TabWidth: 2
+ColumnLimit: 500
+AllowShortCaseLabelsOnASingleLine: true
+AllowShortFunctionsOnASingleLine: true
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
+AlignTrailingComments: false
+SpaceAfterTemplateKeyword: false
+AllowShortBlocksOnASingleLine: true
+MaxEmptyLinesToKeep: 3
+NamespaceIndentation: None
+CommentPragmas: '^[^ ]'
+FixNamespaceComments: false
+AccessModifierOffset: -4
+SpaceAfterCStyleCast: false
+PointerAlignment: Left
+IndentCaseLabels: false
+BinPackArguments: false
+BinPackParameters: false
+Cpp11BracedListStyle: false
+SpaceBeforeCpp11BracedList: true
+SpaceInEmptyBlock: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+DerivePointerAlignment: false
+AlwaysBreakTemplateDeclarations: No
+DeriveLineEnding: false
+UseCRLF: false
+AllowAllArgumentsOnNextLine: true
+AlignAfterOpenBracket: DontAlign
+PackConstructorInitializers: CurrentLine
+BraceWrapping:
+  AfterStruct: false
+  AfterClass: false
+  AfterUnion: false
+  AfterEnum: false
+  AfterControlStatement: false
+  AfterFunction: false
+  AfterNamespace: false
+  AfterExternBlock: false
+  BeforeElse: false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false

package/llama/addon.cpp

@@ -0,0 +1,183 @@
+#include <stddef.h>
+#include <algorithm>
+#include <sstream>
+#include <vector>
+
+#include "llama.h"
+#include "napi.h"
+
+class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
+  public:
+  llama_context_params params;
+  llama_model* model;
+  LLAMAModel(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAModel>(info) {
+    params = llama_context_default_params();
+    params.seed = -1;
+    params.n_ctx = 4096;
+    model = llama_load_model_from_file(info[0].As<Napi::String>().Utf8Value().c_str(), params);
+
+    if (model == NULL) {
+      Napi::Error::New(info.Env(), "Failed to load model").ThrowAsJavaScriptException();
+      return;
+    }
+  }
+  ~LLAMAModel() { llama_free_model(model); }
+  static void init(Napi::Object exports) { exports.Set("LLAMAModel", DefineClass(exports.Env(), "LLAMAModel", {})); }
+};
+
+class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
+  public:
+  LLAMAModel* model;
+  llama_context* ctx;
+  LLAMAContext(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAContext>(info) {
+    model = Napi::ObjectWrap<LLAMAModel>::Unwrap(info[0].As<Napi::Object>());
+    model->Ref();
+    ctx = llama_new_context_with_model(model->model, model->params);
+    Napi::MemoryManagement::AdjustExternalMemory(Env(), llama_get_state_size(ctx));
+  }
+  ~LLAMAContext() {
+    Napi::MemoryManagement::AdjustExternalMemory(Env(), -(int64_t)llama_get_state_size(ctx));
+    llama_free(ctx);
+    model->Unref();
+  }
+  Napi::Value Encode(const Napi::CallbackInfo& info) {
+    std::string text = info[0].As<Napi::String>().Utf8Value();
+
+    std::vector<llama_token> tokens(text.size());
+    int n = llama_tokenize(ctx, text.data(), tokens.data(), text.size(), false);
+
+    if (n < 0) {
+      Napi::Error::New(info.Env(), "String expected").ThrowAsJavaScriptException();
+      return info.Env().Undefined();
+    }
+    tokens.resize(n);
+
+    Napi::Uint32Array result = Napi::Uint32Array::New(info.Env(), n);
+    for (size_t i = 0; i < tokens.size(); ++i) { result[i] = static_cast<uint32_t>(tokens[i]); }
+
+    return result;
+  }
+  Napi::Value Decode(const Napi::CallbackInfo& info) {
+    Napi::Uint32Array tokens = info[0].As<Napi::Uint32Array>();
+
+    // Create a stringstream for accumulating the decoded string.
+    std::stringstream ss;
+
+    // Decode each token and accumulate the result.
+    for (size_t i = 0; i < tokens.ElementLength(); i++) {
+      const char* str = llama_token_to_str(ctx, (llama_token)tokens[i]);
+      if (str == nullptr) {
+        Napi::Error::New(info.Env(), "Invalid token").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+      }
+      ss << str;
+    }
+
+    return Napi::String::New(info.Env(), ss.str());
+  }
+  Napi::Value Eval(const Napi::CallbackInfo& info);
+  static void init(Napi::Object exports) {
+    exports.Set("LLAMAContext",
+        DefineClass(exports.Env(),
+            "LLAMAContext",
+            {
+                InstanceMethod("encode", &LLAMAContext::Encode),
+                InstanceMethod("decode", &LLAMAContext::Decode),
+                InstanceMethod("eval", &LLAMAContext::Eval),
+            }));
+  }
+};
+
+
+class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
+  LLAMAContext* ctx;
+  std::vector<llama_token> tokens;
+  std::vector<llama_token> restriction;
+  llama_token result;
+
+  public:
+  LLAMAContextEvalWorker(const Napi::CallbackInfo& info, LLAMAContext* ctx) : Napi::AsyncWorker(info.Env(), "LLAMAContextEvalWorker"), ctx(ctx), Napi::Promise::Deferred(info.Env()) {
+    ctx->Ref();
+    Napi::Uint32Array tokens = info[0].As<Napi::Uint32Array>();
+    this->tokens.reserve(tokens.ElementLength());
+    for (size_t i = 0; i < tokens.ElementLength(); i++) { this->tokens.push_back(static_cast<llama_token>(tokens[i])); }
+
+    if (info.Length() > 1 && info[1].IsTypedArray()) {
+      Napi::Uint32Array restriction = info[1].As<Napi::Uint32Array>();
+      this->restriction.reserve(restriction.ElementLength());
+      for (size_t i = 0; i < restriction.ElementLength(); i++) { this->restriction.push_back(static_cast<llama_token>(restriction[i])); }
+      std::sort(this->restriction.begin(), this->restriction.end());
+    }
+  }
+  ~LLAMAContextEvalWorker() { ctx->Unref(); }
+  using Napi::AsyncWorker::Queue;
+  using Napi::Promise::Deferred::Promise;
+
+  protected:
+  void Execute() {
+    // Perform the evaluation using llama_eval.
+    int r = llama_eval(ctx->ctx, tokens.data(), tokens.size(), llama_get_kv_cache_token_count(ctx->ctx), 6);
+    if (r != 0) {
+      SetError("Eval has failed");
+      return;
+    }
+
+    // Select the best prediction.
+    float* logits = llama_get_logits(ctx->ctx);
+    int n_vocab = llama_n_vocab(ctx->ctx);
+    llama_token re;
+    if (restriction.empty()) {
+      float max = logits[0];
+      re = 0;
+      for (llama_token id = 1; id < n_vocab; id++) {
+        float logit = logits[id];
+        if (logit > max) {
+          max = logit;
+          re = id;
+        }
+      }
+    } else {
+      float max = logits[restriction[0]];
+      re = 0;
+      for (size_t i = 1; i < restriction.size(); i++) {
+        llama_token id = restriction[i];
+        float logit = logits[id];
+        if (logit > max) {
+          max = logit;
+          re = id;
+        }
+      }
+    }
+    result = re;
+  }
+  void OnOK() {
+    Napi::Env env = Napi::AsyncWorker::Env();
+    Napi::Number resultValue = Napi::Number::New(env, static_cast<uint32_t>(result));
+    Napi::Promise::Deferred::Resolve(resultValue);
+  }
+  void OnError(const Napi::Error& err) { Napi::Promise::Deferred::Reject(err.Value()); }
+};
+
+Napi::Value LLAMAContext::Eval(const Napi::CallbackInfo& info) {
+  LLAMAContextEvalWorker* worker = new LLAMAContextEvalWorker(info, this);
+  worker->Queue();
+  return worker->Promise();
+}
+
+Napi::Value tokenBos(const Napi::CallbackInfo& info) { return Napi::Number::From(info.Env(), llama_token_bos()); }
+Napi::Value tokenEos(const Napi::CallbackInfo& info) { return Napi::Number::From(info.Env(), llama_token_eos()); }
+Napi::Value systemInfo(const Napi::CallbackInfo& info) { return Napi::String::From(info.Env(), llama_print_system_info()); }
+
+Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
+  llama_backend_init(false);
+  exports.DefineProperties({
+      Napi::PropertyDescriptor::Function("tokenBos", tokenBos),
+      Napi::PropertyDescriptor::Function("tokenEos", tokenEos),
+      Napi::PropertyDescriptor::Function("systemInfo", systemInfo),
+  });
+  LLAMAModel::init(exports);
+  LLAMAContext::init(exports);
+  return exports;
+}
+
+NODE_API_MODULE(NODE_GYP_MODULE_NAME, registerCallback)

package/llama/binding.gyp

@@ -0,0 +1,24 @@
+{
+  "targets": [
+    {
+      "target_name": "llama",
+      "sources": [
+        "addon.cpp",
+        "llama.cpp/ggml.c",
+        "llama.cpp/ggml-alloc.c",
+        "llama.cpp/k_quants.c",
+        "llama.cpp/llama.cpp"
+      ],
+      "include_dirs": [
+        "<!@(node -p \"require('node-addon-api').include\")",
+        "llama.cpp"
+      ],
+      "cflags": ["-fexceptions"],
+      "cflags_cc": ["-fexceptions"],
+      "defines": [ "GGML_USE_K_QUANTS", "NAPI_CPP_EXCEPTIONS" ],
+      "msvs_settings": {
+        "VCCLCompilerTool": { "AdditionalOptions": [ '/arch:AVX2', '/EHsc' ] }
+      }
+    }
+  ]
+}

package/llama/usedBin.json

@@ -0,0 +1,3 @@
+{
+    "use": "prebuiltBinaries"
+}