node-llama-cpp 2.8.0 → 3.0.0-beta.1

package/dist/llamaEvaluator/LlamaModel.js

@@ -1,76 +1,311 @@
 import process from "process";
 import path from "path";
+import { DisposedError, EventRelay } from "lifecycle-utils";
 import { removeNullFields } from "../utils/removeNullFields.js";
-import { llamaCppNode, LLAMAModel } from "./LlamaBins.js";
+import { addonBinding, AddonModel } from "./LlamaBins.js";
 export class LlamaModel {
-    /** @internal */
-    _model;
-    /** @internal */
-    _contextOptions;
-    /** @internal */
-    _evaluationOptions;
+    /** @internal */ _model;
+    /** @internal */ _tokens;
+    /** @internal */ _filename;
+    /** @internal */ _disposedState = { disposed: false };
+    /** @internal */ _typeDescription;
+    /** @internal */ _trainContextSize;
+    onDispose = new EventRelay();
     /**
      * > options source:
      * > [github:ggerganov/llama.cpp/llama.h](
-     * > https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/llama.h#L102) (`struct llama_context_params`)
-     * @param {object} options
-     * @param {string} options.modelPath - path to the model on the filesystem
-     * @param {number | null} [options.seed] - If null, a random seed will be used
-     * @param {number} [options.contextSize] - text context size
-     * @param {number} [options.batchSize] - prompt processing batch size
-     * @param {number} [options.gpuLayers] - number of layers to store in VRAM
-     * @param {number} [options.threads] - number of threads to use to evaluate tokens
-     * @param {number} [options.temperature] - Temperature is a hyperparameter that controls the randomness of the generated text.
-     * It affects the probability distribution of the model's output tokens.
-     * A higher temperature (e.g., 1.5) makes the output more random and creative,
-     * while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative.
-     * The suggested temperature is 0.8, which provides a balance between randomness and determinism.
-     * At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.
-     *
-     * Set to `0` to disable.
-     * @param {number} [options.topK] - Limits the model to consider only the K most likely next tokens for sampling at each step of
-     * sequence generation.
-     * An integer number between `1` and the size of the vocabulary.
-     * Set to `0` to disable (which uses the full vocabulary).
-     *
-     * Only relevant when `temperature` is set to a value greater than 0.
-     * @param {number} [options.topP] - Dynamically selects the smallest set of tokens whose cumulative probability exceeds the threshold P,
-     * and samples the next token only from this set.
-     * A float number between `0` and `1`.
-     * Set to `1` to disable.
-     *
-     * Only relevant when `temperature` is set to a value greater than `0`.
-     * @param {boolean} [options.f16Kv] - use fp16 for KV cache
-     * @param {boolean} [options.logitsAll] - the llama_eval() call computes all logits, not just the last one
-     * @param {boolean} [options.vocabOnly] - only load the vocabulary, no weights
-     * @param {boolean} [options.useMmap] - use mmap if possible
-     * @param {boolean} [options.useMlock] - force system to keep model in RAM
-     * @param {boolean} [options.embedding] - embedding mode only
-     */
-    constructor({ modelPath, seed = null, contextSize = 1024 * 4, batchSize, gpuLayers, threads = 6, temperature = 0, topK = 40, topP = 0.95, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }) {
-        this._model = new LLAMAModel(path.resolve(process.cwd(), modelPath), removeNullFields({
+     * > https://github.com/ggerganov/llama.cpp/blob/05816027d649f977468fc804cdb54e99eac246d1/llama.h#L161) (`struct llama_model_params`)
+     * @param options
+     * @param options.modelPath - path to the model on the filesystem
+     * @param [options.gpuLayers] - number of layers to store in VRAM
+     * @param [options.vocabOnly] - only load the vocabulary, no weights
+     * @param [options.useMmap] - use mmap if possible
+     * @param [options.useMlock] - force system to keep model in RAM
+     */
+    constructor({ modelPath, gpuLayers, vocabOnly, useMmap, useMlock }) {
+        this._model = new AddonModel(path.resolve(process.cwd(), modelPath), removeNullFields({
             gpuLayers,
             vocabOnly,
             useMmap,
             useMlock
         }));
-        this._contextOptions = {
-            seed,
-            contextSize,
-            batchSize,
-            f16Kv,
-            logitsAll,
-            embedding,
-            threads
-        };
-        this._evaluationOptions = {
-            temperature,
-            topK,
-            topP
-        };
+        this._tokens = LlamaModelTokens._create(this._model, this._disposedState);
+        this._filename = path.basename(modelPath);
+    }
+    dispose() {
+        if (this._disposedState.disposed)
+            return;
+        this.onDispose.dispatchEvent();
+        this._model.dispose();
+        this._disposedState.disposed = true;
+    }
+    /** @hidden */
+    [Symbol.dispose]() {
+        this.dispose();
+    }
+    get disposed() {
+        return this._disposedState.disposed;
+    }
+    get tokens() {
+        return this._tokens;
+    }
+    get filename() {
+        return this._filename;
+    }
+    /** Transform text into tokens that can be fed to the model */
+    tokenize(text) {
+        this._ensureNotDisposed();
+        if (text === "")
+            return [];
+        return Array.from(this._model.tokenize(text));
+    }
+    /** Transform tokens into text */
+    detokenize(tokens) {
+        this._ensureNotDisposed();
+        if (tokens.length === 0)
+            return "";
+        return this._model.detokenize(Uint32Array.from(tokens));
+    }
+    /** @hidden `ModelTypeDescription` type alias is too long in the documentation */
+    get typeDescription() {
+        this._ensureNotDisposed();
+        if (this._typeDescription == null)
+            this._typeDescription = this._model.getModelDescription();
+        return this._typeDescription;
+    }
+    /** The context size the model was trained on */
+    get trainContextSize() {
+        this._ensureNotDisposed();
+        if (this._trainContextSize == null)
+            this._trainContextSize = this._model.getTrainContextSize();
+        return this._trainContextSize;
+    }
+    /** @internal */
+    _ensureNotDisposed() {
+        if (this._disposedState.disposed)
+            throw new DisposedError();
     }
     static get systemInfo() {
-        return llamaCppNode.systemInfo();
+        return addonBinding.systemInfo();
+    }
+}
+export class LlamaModelTokens {
+    /** @internal */ _model;
+    /** @internal */ _disposedState;
+    /** @internal */ _infillTokens;
+    /** @internal */ _bosToken;
+    /** @internal */ _eosToken;
+    /** @internal */ _nlToken;
+    /** @internal */ _bosString;
+    /** @internal */ _eosString;
+    /** @internal */ _nlString;
+    constructor(model, disposedState) {
+        this._model = model;
+        this._disposedState = disposedState;
+    }
+    /**
+     * @returns infill tokens
+     */
+    get infill() {
+        this._ensureNotDisposed();
+        if (this._infillTokens == null)
+            this._infillTokens = LlamaModelInfillTokens._create(this._model, this._disposedState);
+        return this._infillTokens;
+    }
+    /**
+     * @returns The BOS (Beginning Of Sequence) token.
+     */
+    get bos() {
+        this._ensureNotDisposed();
+        if (this._bosToken == null)
+            this._bosToken = this._model.tokenBos();
+        if (this._bosToken === -1)
+            return null;
+        return this._bosToken;
+    }
+    /**
+     * @returns The EOS (End Of Sequence) token.
+     */
+    get eos() {
+        this._ensureNotDisposed();
+        if (this._eosToken == null)
+            this._eosToken = this._model.tokenEos();
+        if (this._eosToken === -1)
+            return null;
+        return this._eosToken;
+    }
+    /**
+     * @returns The NL (New Line) token.
+     */
+    get nl() {
+        this._ensureNotDisposed();
+        if (this._nlToken == null)
+            this._nlToken = this._model.tokenNl();
+        if (this._nlToken === -1)
+            return null;
+        return this._nlToken;
+    }
+    /**
+     * @returns The BOS (Beginning Of Sequence) token as a string.
+     */
+    get bosString() {
+        this._ensureNotDisposed();
+        const bosToken = this.bos;
+        if (bosToken == null)
+            return null;
+        if (this._bosString == null)
+            this._bosString = this._model.getTokenString(bosToken);
+        return this._bosString;
+    }
+    /**
+     * @returns The EOS (End Of Sequence) token as a string.
+     */
+    get eosString() {
+        this._ensureNotDisposed();
+        const eosToken = this.eos;
+        if (eosToken == null)
+            return null;
+        if (this._eosString == null)
+            this._eosString = this._model.getTokenString(eosToken);
+        return this._eosString;
+    }
+    /**
+     * @returns The NL (New Line) token as a string.
+     */
+    get nlString() {
+        this._ensureNotDisposed();
+        const nlToken = this.nl;
+        if (nlToken == null)
+            return null;
+        if (this._nlString == null)
+            this._nlString = this._model.getTokenString(nlToken);
+        return this._nlString;
+    }
+    /** @internal */
+    _ensureNotDisposed() {
+        if (this._disposedState.disposed)
+            throw new DisposedError();
+    }
+    /** @internal */
+    static _create(model, disposedState) {
+        return new LlamaModelTokens(model, disposedState);
+    }
+}
+export class LlamaModelInfillTokens {
+    /** @internal */ _model;
+    /** @internal */ _disposedState;
+    /** @internal */ _prefixToken;
+    /** @internal */ _middleToken;
+    /** @internal */ _suffixToken;
+    /** @internal */ _eotToken;
+    /** @internal */ _prefixString;
+    /** @internal */ _middleString;
+    /** @internal */ _suffixString;
+    /** @internal */ _eotString;
+    constructor(model, disposedState) {
+        this._model = model;
+        this._disposedState = disposedState;
+    }
+    /**
+     * @returns The beginning of infill prefix token.
+     */
+    get prefix() {
+        this._ensureNotDisposed();
+        if (this._prefixToken == null)
+            this._prefixToken = this._model.prefixToken();
+        if (this._prefixToken === -1)
+            return null;
+        return this._prefixToken;
+    }
+    /**
+     * @returns The beginning of infill middle token.
+     */
+    get middle() {
+        this._ensureNotDisposed();
+        if (this._middleToken == null)
+            this._middleToken = this._model.middleToken();
+        if (this._middleToken === -1)
+            return null;
+        return this._middleToken;
+    }
+    /**
+     * @returns The beginning of infill suffix token.
+     */
+    get suffix() {
+        this._ensureNotDisposed();
+        if (this._suffixToken == null)
+            this._suffixToken = this._model.suffixToken();
+        if (this._suffixToken === -1)
+            return null;
+        return this._suffixToken;
+    }
+    /**
+     * @returns End of infill middle token (End Of Text).
+     */
+    get eot() {
+        this._ensureNotDisposed();
+        if (this._eotToken == null)
+            this._eotToken = this._model.eotToken();
+        if (this._eotToken === -1)
+            return null;
+        return this._eotToken;
+    }
+    /**
+     * @returns The beginning of infill prefix token as a string.
+     */
+    get prefixString() {
+        this._ensureNotDisposed();
+        const prefixToken = this.prefix;
+        if (prefixToken == null)
+            return null;
+        if (this._prefixString == null)
+            this._prefixString = this._model.getTokenString(prefixToken);
+        return this._prefixString;
+    }
+    /**
+     * @returns The beginning of infill middle token as a string.
+     */
+    get middleString() {
+        this._ensureNotDisposed();
+        const middleToken = this.middle;
+        if (middleToken == null)
+            return null;
+        if (this._middleString == null)
+            this._middleString = this._model.getTokenString(middleToken);
+        return this._middleString;
+    }
+    /**
+     * @returns The beginning of infill suffix token as a string.
+     */
+    get suffixString() {
+        this._ensureNotDisposed();
+        const suffixToken = this.suffix;
+        if (suffixToken == null)
+            return null;
+        if (this._suffixString == null)
+            this._suffixString = this._model.getTokenString(suffixToken);
+        return this._suffixString;
+    }
+    /**
+     * @returns End of infill middle token (End Of Text) as a string.
+     */
+    get eotString() {
+        this._ensureNotDisposed();
+        const eotToken = this.eot;
+        if (eotToken == null)
+            return null;
+        if (this._eotString == null)
+            this._eotString = this._model.getTokenString(eotToken);
+        return this._eotString;
+    }
+    /** @internal */
+    _ensureNotDisposed() {
+        if (this._disposedState.disposed)
+            throw new DisposedError();
+    }
+    /** @internal */
+    static _create(model, disposedState) {
+        return new LlamaModelInfillTokens(model, disposedState);
     }
 }
 //# sourceMappingURL=LlamaModel.js.map

package/dist/llamaEvaluator/LlamaModel.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,gBAAgB,CAAC;AA0GxD,MAAM,OAAO,UAAU;IACnB,gBAAgB;IACA,MAAM,CAAa;IAEnC,gBAAgB;IACA,eAAe,CAQ7B;IAEF,gBAAgB;IACA,kBAAkB,CAIhC;IAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqCG;IACH,YAAmB,EACf,SAAS,EAAE,IAAI,GAAG,IAAI,EAAE,WAAW,GAAG,IAAI,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,EACpE,OAAO,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,EAAE,IAAI,GAAG,EAAE,EAAE,IAAI,GAAG,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAC/F;QAChB,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,CAAC,EAAE,gBAAgB,CAAC;YAClF,SAAS;YACT,SAAS;YACT,OAAO;YACP,QAAQ;SACX,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,eAAe,GAAG;YACnB,IAAI;YACJ,WAAW;YACX,SAAS;YACT,KAAK;YACL,SAAS;YACT,SAAS;YACT,OAAO;SACV,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG;YACtB,WAAW;YACX,IAAI;YACJ,IAAI;SACP,CAAC;IACN,CAAC;IAEM,MAAM,KAAK,UAAU;QACxB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;CACJ"}
+{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAC,aAAa,EAAE,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAG9D,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAoBxD,MAAM,OAAO,UAAU;IACnB,gBAAgB,CAAiB,MAAM,CAAa;IACpD,gBAAgB,CAAkB,OAAO,CAAmB;IAC5D,gBAAgB,CAAkB,SAAS,CAAU;IACrD,gBAAgB,CAAkB,cAAc,GAAkB,EAAC,QAAQ,EAAE,KAAK,EAAC,CAAC;IACpF,gBAAgB,CAAS,gBAAgB,CAAwB;IACjE,gBAAgB,CAAS,iBAAiB,CAAU;IAEpC,SAAS,GAAG,IAAI,UAAU,EAAQ,CAAC;IAEnD;;;;;;;;;;OAUG;IACH,YAAmB,EACf,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAClC;QAChB,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,CAAC,EAAE,gBAAgB,CAAC;YAClF,SAAS;YACT,SAAS;YACT,OAAO;YACP,QAAQ;SACX,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAC1E,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IAEM,OAAO;QACV,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ;YAC5B,OAAO;QAEX,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC;QAC/B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACtB,IAAI,CAAC,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC;IACxC,CAAC;IAED,cAAc;IACP,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;IACxC,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,8DAA8D;IACvD,QAAQ,CAAC,IAAY;QACxB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,KAAK,EAAE;YACX,OAAO,EAAE,CAAC;QAEd,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAY,CAAC;IAC7D,CAAC;IAED,iCAAiC;IAC1B,UAAU,CAAC,MAAwB;QACtC,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACnB,OAAO,EAAE,CAAC;QAEd,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,iFAAiF;IACjF,IAAW,eAAe;QACtB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI;YAC7B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAE9D,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAED,gDAAgD;IAChD,IAAW,gBAAgB;QACvB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI;YAC9B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAE/D,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAClC,CAAC;IAED,gBAAgB;IACR,kBAAkB;QACtB,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ;YAC5B,MAAM,IAAI,aAAa,EAAE,CAAC;IAClC,CAAC;IAEM,MAAM,KAAK,UAAU;QACxB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;CACJ;AAED,MAAM,OAAO,gBAAgB;IACzB,gBAAgB,CAAkB,MAAM,CAAa;IACrD,gBAAgB,CAAkB,cAAc,CAAgB;IAChE,gBAAgB,CAAS,aAAa,CAA0B;IAChE,gBAAgB,CAAS,SAAS,CAAS;IAC3C,gBAAgB,CAAS,SAAS,CAAS;IAC3C,gBAAgB,CAAS,QAAQ,CAAS;IAC1C,gBAAgB,CAAS,UAAU,CAAU;IAC7C,gBAAgB,CAAS,UAAU,CAAU;IAC7C,gBAAgB,CAAS,SAAS,CAAU;IAE5C,YAAoB,KAAiB,EAAE,aAA4B;QAC/D,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,IAAW,MAAM;QACb,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI;YAC1B,IAAI,CAAC,aAAa,GAAG,sBAAsB,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAE1F,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,IAAW,GAAG;QACV,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI;YACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAE5C,IAAI,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC;YACrB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,IAAW,GAAG;QACV,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI;YACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAE5C,IAAI,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC;YACrB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,IAAW,EAAE;QACT,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI;YACrB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAE1C,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC;YACpB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,IAAW,SAAS;QAChB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;QAE1B,IAAI,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC;QAEhB,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI;YACvB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAE3D,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,IAAW,SAAS;QAChB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;QAE1B,IAAI,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC;QAEhB,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI;YACvB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAE3D,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,IAAW,QAAQ;QACf,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC;QAExB,IAAI,OAAO,IAAI,IAAI;YACf,OAAO,IAAI,CAAC;QAEhB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI;YACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAEzD,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,gBAAgB;IACR,kBAAkB;QACtB,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ;YAC5B,MAAM,IAAI,aAAa,EAAE,CAAC;IAClC,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,KAAiB,EAAE,aAA4B;QACjE,OAAO,IAAI,gBAAgB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;IACtD,CAAC;CACJ;AAED,MAAM,OAAO,sBAAsB;IAC/B,gBAAgB,CAAkB,MAAM,CAAa;IACrD,gBAAgB,CAAkB,cAAc,CAAgB;IAChE,gBAAgB,CAAS,YAAY,CAAS;IAC9C,gBAAgB,CAAS,YAAY,CAAS;IAC9C,gBAAgB,CAAS,YAAY,CAAS;IAC9C,gBAAgB,CAAS,SAAS,CAAS;IAC3C,gBAAgB,CAAS,aAAa,CAAU;IAChD,gBAAgB,CAAS,aAAa,CAAU;IAChD,gBAAgB,CAAS,aAAa,CAAU;IAChD,gBAAgB,CAAS,UAAU,CAAU;IAE7C,YAAoB,KAAiB,EAAE,aAA4B;QAC/D,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,IAAW,MAAM;QACb,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI;YACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAElD,IAAI,IAAI,CAAC,YAAY,KAAK,CAAC,CAAC;YACxB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,IAAW,MAAM;QACb,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI;YACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAElD,IAAI,IAAI,CAAC,YAAY,KAAK,CAAC,CAAC;YACxB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,IAAW,MAAM;QACb,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI;YACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAElD,IAAI,IAAI,CAAC,YAAY,KAAK,CAAC,CAAC;YACxB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,IAAW,GAAG;QACV,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI;YACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAE5C,IAAI,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC;YACrB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,IAAW,YAAY;QACnB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC;QAEhC,IAAI,WAAW,IAAI,IAAI;YACnB,OAAO,IAAI,CAAC;QAEhB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI;YAC1B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEjE,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,IAAW,YAAY;QACnB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC;QAEhC,IAAI,WAAW,IAAI,IAAI;YACnB,OAAO,IAAI,CAAC;QAEhB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI;YAC1B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEjE,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,IAAW,YAAY;QACnB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC;QAEhC,IAAI,WAAW,IAAI,IAAI;YACnB,OAAO,IAAI,CAAC;QAEhB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI;YAC1B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEjE,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,IAAW,SAAS;QAChB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;QAE1B,IAAI,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC;QAEhB,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI;YACvB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAE3D,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,gBAAgB;IACR,kBAAkB;QACtB,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ;YAC5B,MAAM,IAAI,aAAa,EAAE,CAAC;IAClC,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,KAAiB,EAAE,aAA4B;QACjE,OAAO,IAAI,sBAAsB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;IAC5D,CAAC;CACJ"}

package/dist/types.d.ts

@@ -1,4 +1,6 @@
-export type Token = number;
+export type Token = number & {
+    __token: never;
+};
 export type ConversationInteraction = {
     prompt: string;
     response: string;

package/dist/utils/ReplHistory.js

@@ -1,5 +1,5 @@
 import fs from "fs-extra";
-import { withLock } from "./withLock.js";
+import { withLock } from "lifecycle-utils";
 const emptyHistory = {
     history: []
 };

package/dist/utils/ReplHistory.js.map

@@ -1 +1 @@
-{"version":3,"file":"ReplHistory.js","sourceRoot":"","sources":["../../src/utils/ReplHistory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,QAAQ,EAAC,MAAM,eAAe,CAAC;AAMvC,MAAM,YAAY,GAAqB;IACnC,OAAO,EAAE,EAAE;CACd,CAAC;AAEF,MAAM,OAAO,WAAW;IACH,SAAS,CAAgB;IAClC,YAAY,CAAmB;IAEvC,YAAoB,QAAuB,EAAE,WAA6B;QACtE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,IAAY;QACzB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;YACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YACpE,OAAO;SACV;QAED,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI;gBACA,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAU,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAEvD,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAU,EAAE,IAAI,CAAC,YAAY,EAAE;oBACnD,MAAM,EAAE,CAAC;iBACZ,CAAC,CAAC;aACN;YAAC,OAAO,GAAG,EAAE,GAAE;QACpB,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IACrC,CAAC;IAEO,iBAAiB,CAAC,IAAY,EAAE,WAA6B;QACjE,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/C,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAElD,IAAI,gBAAgB,KAAK,CAAC,CAAC;YACvB,UAAU,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;QAE3C,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAEzB,OAAO;YACH,GAAG,WAAW;YACd,OAAO,EAAE,UAAU;SACtB,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAgB,EAAE,qBAA8B,IAAI;QACzE,IAAI,CAAC,kBAAkB;YACnB,OAAO,IAAI,WAAW,CAAC,IAAI,EAAE;gBACzB,OAAO,EAAE,EAAE;aACd,CAAC,CAAC;QAEP,IAAI;YACA,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAChC,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE;oBACvC,MAAM,EAAE,CAAC;iBACZ,CAAC,CAAC;YAEP,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC5D,OAAO,IAAI,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;SAC1C;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,IAAI,WAAW,CAAC,IAAI,EAAE;gBACzB,OAAO,EAAE,EAAE;aACd,CAAC,CAAC;SACN;IACL,CAAC;CACJ;AAED,SAAS,iBAAiB,CAAC,IAAa;IACpC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,YAAY,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;QAC7J,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAEjD,OAAO,IAAwB,CAAC;AACpC,CAAC"}
+{"version":3,"file":"ReplHistory.js","sourceRoot":"","sources":["../../src/utils/ReplHistory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AAMzC,MAAM,YAAY,GAAqB;IACnC,OAAO,EAAE,EAAE;CACd,CAAC;AAEF,MAAM,OAAO,WAAW;IACH,SAAS,CAAgB;IAClC,YAAY,CAAmB;IAEvC,YAAoB,QAAuB,EAAE,WAA6B;QACtE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,IAAY;QACzB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;YACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YACpE,OAAO;SACV;QAED,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI;gBACA,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAU,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAEvD,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAU,EAAE,IAAI,CAAC,YAAY,EAAE;oBACnD,MAAM,EAAE,CAAC;iBACZ,CAAC,CAAC;aACN;YAAC,OAAO,GAAG,EAAE,GAAE;QACpB,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IACrC,CAAC;IAEO,iBAAiB,CAAC,IAAY,EAAE,WAA6B;QACjE,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/C,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAElD,IAAI,gBAAgB,KAAK,CAAC,CAAC;YACvB,UAAU,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;QAE3C,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAEzB,OAAO;YACH,GAAG,WAAW;YACd,OAAO,EAAE,UAAU;SACtB,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAgB,EAAE,qBAA8B,IAAI;QACzE,IAAI,CAAC,kBAAkB;YACnB,OAAO,IAAI,WAAW,CAAC,IAAI,EAAE;gBACzB,OAAO,EAAE,EAAE;aACd,CAAC,CAAC;QAEP,IAAI;YACA,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAChC,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE;oBACvC,MAAM,EAAE,CAAC;iBACZ,CAAC,CAAC;YAEP,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC5D,OAAO,IAAI,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;SAC1C;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,IAAI,WAAW,CAAC,IAAI,EAAE;gBACzB,OAAO,EAAE,EAAE;aACd,CAAC,CAAC;SACN;IACL,CAAC;CACJ;AAED,SAAS,iBAAiB,CAAC,IAAa;IACpC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,YAAY,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;QAC7J,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAEjD,OAAO,IAAwB,CAAC;AACpC,CAAC"}

package/dist/utils/getBin.d.ts

@@ -1,32 +1,60 @@
+import { Token } from "../types.js";
 export declare function getPrebuildBinPath(): Promise<string | null>;
-export declare function loadBin(): Promise<LlamaCppNodeModule>;
-export type LlamaCppNodeModule = {
-    LLAMAModel: LLAMAModel;
-    LLAMAContext: LLAMAContext;
-    LLAMAGrammar: LLAMAGrammar;
-    LLAMAGrammarEvaluationState: LLAMAGrammarEvaluationState;
+export declare function loadBin(): Promise<BindingModule>;
+export type BindingModule = {
+    AddonModel: {
+        new (modelPath: string, params: {
+            gpuLayers?: number;
+            vocabOnly?: boolean;
+            useMmap?: boolean;
+            useMlock?: boolean;
+        }): AddonModel;
+    };
+    AddonContext: {
+        new (model: AddonModel, params: {
+            seed?: number;
+            contextSize?: number;
+            batchSize?: number;
+            f16Kv?: boolean;
+            logitsAll?: boolean;
+            embedding?: boolean;
+            threads?: number;
+        }): AddonContext;
+    };
+    AddonGrammar: {
+        new (grammarPath: string, params?: {
+            printGrammar?: boolean;
+        }): AddonGrammar;
+    };
+    AddonGrammarEvaluationState: {
+        new (grammar: AddonGrammar): AddonGrammarEvaluationState;
+    };
     systemInfo(): string;
 };
-export type LLAMAModel = {
-    new (modelPath: string, params: {
-        gpuLayers?: number;
-        vocabOnly?: boolean;
-        useMmap?: boolean;
-        useMlock?: boolean;
-    }): LLAMAModel;
+export type AddonModel = {
+    dispose(): void;
+    tokenize(text: string): Uint32Array;
+    detokenize(tokens: Uint32Array): string;
+    getTrainContextSize(): number;
+    getTotalSize(): number;
+    getTotalParameters(): number;
+    getModelDescription(): ModelTypeDescription;
+    tokenBos(): Token;
+    tokenEos(): Token;
+    tokenNl(): Token;
+    prefixToken(): Token;
+    middleToken(): Token;
+    suffixToken(): Token;
+    eotToken(): Token;
+    getTokenString(token: number): string;
 };
-export type LLAMAContext = {
-    new (model: LLAMAModel, params: {
-        seed?: number;
-        contextSize?: number;
-        batchSize?: number;
-        f16Kv?: boolean;
-        logitsAll?: boolean;
-        embedding?: boolean;
-        threads?: number;
-    }): LLAMAContext;
-    encode(text: string): Uint32Array;
-    eval(tokens: Uint32Array, options?: {
+export type AddonContext = {
+    dispose(): void;
+    getContextSize(): number;
+    initBatch(size: number): void;
+    addToBatch(sequenceId: number, firstTokenSequenceIndex: number, tokens: Uint32Array, generateLogitAtTheEnd: boolean): BatchLogitIndex | undefined;
+    decodeBatch(): Promise<void>;
+    sampleToken(batchLogitIndex: BatchLogitIndex, options?: {
         temperature?: number;
         topK?: number;
         topP?: number;
@@ -34,20 +62,24 @@ export type LLAMAContext = {
         repeatPenaltyTokens?: Uint32Array;
         repeatPenaltyPresencePenalty?: number;
         repeatPenaltyFrequencyPenalty?: number;
-        grammarEvaluationState?: LLAMAGrammarEvaluationState;
-    }): Promise<number>;
-    decode(tokens: Uint32Array): string;
-    tokenBos(): number;
-    tokenEos(): number;
-    tokenNl(): number;
-    getContextSize(): number;
-    getTokenString(token: number): string;
+        grammarEvaluationState?: AddonGrammarEvaluationState;
+    }): Promise<Token>;
+    disposeSequence(sequenceId: number): void;
+    removeTokenCellsFromSequence(sequenceId: number, startPos: number, endPos: number): void;
+    shiftSequenceTokenCells(sequenceId: number, startPos: number, endPos: number, shiftDelta: number): void;
+};
+export type BatchLogitIndex = number & {
+    __batchLogitIndex: never;
 };
-export type LLAMAGrammar = {
-    new (grammarPath: string, params?: {
-        printGrammar?: boolean;
-    }): LLAMAGrammar;
+export type AddonGrammar = "AddonGrammar" & {
+    __brand: never;
 };
-export type LLAMAGrammarEvaluationState = {
-    new (grammar: LLAMAGrammar): LLAMAGrammarEvaluationState;
+export type AddonGrammarEvaluationState = "AddonGrammarEvaluationState" & {
+    __brand: never;
 };
+export type ModelTypeDescription = `${AddonModelArchName} ${AddonModelTypeName} ${AddonModelFileTypeName}`;
+export type AddonModelArchName = "unknown" | "llama" | "falcon" | "gpt2" | "gptj" | "gptneox" | "mpt" | "baichuan" | "starcoder" | "persimmon" | "refact" | "bloom" | "stablelm";
+export type AddonModelTypeName = "1B" | "3B" | "7B" | "8B" | "13B" | "15B" | "30B" | "34B" | "40B" | "65B" | "70B" | "?B";
+export type AddonModelFileTypeName = _AddonModelFileTypeName | `${_AddonModelFileTypeName} (guessed)`;
+type _AddonModelFileTypeName = "all F32" | "mostly F16" | "mostly Q4_0" | "mostly Q4_1" | "mostly Q4_1, some F16" | "mostly Q5_0" | "mostly Q5_1" | "mostly Q8_0" | "mostly Q2_K" | "mostly Q3_K - Small" | "mostly Q3_K - Medium" | "mostly Q3_K - Large" | "mostly Q4_K - Small" | "mostly Q4_K - Medium" | "mostly Q5_K - Small" | "mostly Q5_K - Medium" | "mostly Q6_K" | "unknown, may not work";
+export {};

package/dist/utils/getBin.js.map

@@ -1 +1 @@
-{"version":3,"file":"getBin.js","sourceRoot":"","sources":["../../src/utils/getBin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EACH,0BAA0B,EAAE,yBAAyB,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,mBAAmB,EAC/H,kBAAkB,EACrB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,6BAA6B,EAAC,MAAM,sBAAsB,CAAC;AAEnE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACpC,SAAS,UAAU,CAAC,QAAgB,EAAE,IAAY;QAC9C,OAAO,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,QAAQ,IAAI,IAAI,mBAAmB,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,IAAY;QACrD,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAE3C,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;YAC5B,OAAO,OAAO,CAAC;QAEnB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,UAAU,OAAO;QAClB,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,OAAO,CAAC;YACb,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YAE5C,KAAK,OAAO,CAAC;YACb,KAAK,SAAS;gBACV,OAAO,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YAE9C,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;SAC/C;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IACzB,MAAM,WAAW,GAAG,MAAM,cAAc,EAAE,CAAC;IAE3C,IAAI,WAAW,KAAK,kBAAkB,EAAE;QACpC,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAEnD,IAAI,eAAe,IAAI,IAAI,EAAE;YACzB,OAAO,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;SAC1F;aAAM;YACH,IAAI;gBACA,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;aACnC;YAAC,OAAO,GAAG,EAAE;gBACV,OAAO,CAAC,KAAK,CAAC,gDAAgD,OAAO,CAAC,QAAQ,MAAM,OAAO,CAAC,IAAI,WAAW,EAAE,GAAG,CAAC,CAAC;gBAClH,OAAO,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;gBAEvD,IAAI;oBACA,OAAO,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC;iBAC1D;gBAAC,OAAO,GAAG,EAAE,GAAE;aACnB;SACJ;KACJ;IAED,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;IAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;QACpB,IAAI,mBAAmB,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;SACzG;aAAM;YACH,MAAM,uBAAuB,CAAC;gBAC1B,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,sBAAsB;gBAC/B,KAAK,EAAE,2BAA2B;gBAClC,IAAI,EAAE,0BAA0B;aACnC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;YAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;aAC/D;YAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;SAC9B;KACJ;IAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;AAC/B,CAAC"}
+{"version":3,"file":"getBin.js","sourceRoot":"","sources":["../../src/utils/getBin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EACH,0BAA0B,EAAE,yBAAyB,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,mBAAmB,EAC/H,kBAAkB,EACrB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAE3E,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,6BAA6B,EAAC,MAAM,sBAAsB,CAAC;AAEnE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACpC,SAAS,UAAU,CAAC,QAAgB,EAAE,IAAY;QAC9C,OAAO,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,QAAQ,IAAI,IAAI,mBAAmB,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,IAAY;QACrD,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAE3C,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;YAC5B,OAAO,OAAO,CAAC;QAEnB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,UAAU,OAAO;QAClB,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,OAAO,CAAC;YACb,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YAE5C,KAAK,OAAO,CAAC;YACb,KAAK,SAAS;gBACV,OAAO,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YAE9C,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;SAC/C;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IACzB,MAAM,WAAW,GAAG,MAAM,cAAc,EAAE,CAAC;IAE3C,IAAI,WAAW,KAAK,kBAAkB,EAAE;QACpC,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAEnD,IAAI,eAAe,IAAI,IAAI,EAAE;YACzB,OAAO,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;SAC1F;aAAM;YACH,IAAI;gBACA,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;aACnC;YAAC,OAAO,GAAG,EAAE;gBACV,OAAO,CAAC,KAAK,CAAC,gDAAgD,OAAO,CAAC,QAAQ,MAAM,OAAO,CAAC,IAAI,WAAW,EAAE,GAAG,CAAC,CAAC;gBAClH,OAAO,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;gBAEvD,IAAI;oBACA,OAAO,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC;iBAC1D;gBAAC,OAAO,GAAG,EAAE,GAAE;aACnB;SACJ;KACJ;IAED,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;IAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;QACpB,IAAI,mBAAmB,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;SACzG;aAAM;YACH,MAAM,uBAAuB,CAAC;gBAC1B,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,sBAAsB;gBAC/B,KAAK,EAAE,2BAA2B;gBAClC,IAAI,EAAE,0BAA0B;aACnC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;YAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;aAC/D;YAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;SAC9B;KACJ;IAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;AAC/B,CAAC"}

package/dist/utils/getReleaseInfo.d.ts

@@ -1,7 +1,7 @@
 export declare function getReleaseInfo(): Promise<{
+    moduleVersion: string;
     llamaCpp: {
         binarySource: string;
         release: string;
     };
-    moduleVersion: string;
 }>;

package/dist/utils/getReleaseInfo.js.map

@@ -1 +1 @@
-{"version":3,"file":"getReleaseInfo.js","sourceRoot":"","sources":["../../src/utils/getReleaseInfo.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,+BAA+B,EAAC,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAC,wBAAwB,EAAC,MAAM,4BAA4B,CAAC;AAEpE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,KAAK,UAAU,cAAc;IAChC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACnD,cAAc,EAAE;QAChB,gBAAgB,EAAE;KACrB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,WAAW,KAAK,kBAAkB;QAC9C,CAAC,CAAC,MAAM,wBAAwB,EAAE;QAClC,CAAC,CAAC,CAAC,MAAM,+BAA+B,EAAE,IAAI,MAAM,wBAAwB,EAAE,CAAC,CAAC;IAEpF,OAAO;QACH,QAAQ,EAAE;YACN,YAAY,EAAE,WAAW,KAAK,kBAAkB;gBAC5C,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,cAAc;YACpB,OAAO;SACV;QACD,aAAa;KAChB,CAAC;AACN,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC3B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;IAExF,OAAO,WAAW,CAAC,OAAO,CAAC;AAC/B,CAAC"}
+{"version":3,"file":"getReleaseInfo.js","sourceRoot":"","sources":["../../src/utils/getReleaseInfo.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,+BAA+B,EAAC,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAC,wBAAwB,EAAC,MAAM,4BAA4B,CAAC;AAEpE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,KAAK,UAAU,cAAc;IAOhC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACnD,cAAc,EAAE;QAChB,gBAAgB,EAAE;KACrB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,WAAW,KAAK,kBAAkB;QAC9C,CAAC,CAAC,MAAM,wBAAwB,EAAE;QAClC,CAAC,CAAC,CAAC,MAAM,+BAA+B,EAAE,IAAI,MAAM,wBAAwB,EAAE,CAAC,CAAC;IAEpF,OAAO;QACH,QAAQ,EAAE;YACN,YAAY,EAAE,WAAW,KAAK,kBAAkB;gBAC5C,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,cAAc;YACpB,OAAO;SACV;QACD,aAAa;KAChB,CAAC;AACN,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC3B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;IAExF,OAAO,WAAW,CAAC,OAAO,CAAC;AAC/B,CAAC"}

package/dist/utils/parseModelFileName.d.ts

@@ -0,0 +1,9 @@
+export declare function parseModelFileName(filename: string): {
+    name: string | undefined;
+    subType: string;
+    quantization: string | undefined;
+    fileType: string | undefined;
+    version: string | undefined;
+    contextSize: string | undefined;
+    parameters: `${number}B` | undefined;
+};

package/dist/utils/parseModelFileName.js

@@ -0,0 +1,68 @@
+export function parseModelFileName(filename) {
+    const parts = filename.split("-");
+    let quantization;
+    let fileType;
+    let version;
+    let contextSize;
+    if (parts.length > 0) {
+        const lastPart = parts[parts.length - 1];
+        const lastParts = lastPart.split(".");
+        fileType = lastParts.pop();
+        quantization = lastParts.pop();
+        if (lastParts.length > 0)
+            parts[parts.length - 1] = lastParts.join(".");
+        else
+            parts.pop();
+    }
+    const { previousParts, parameters, nextParts } = splitByModelParameters(parts);
+    const name = previousParts.shift();
+    for (let i = 0; i < nextParts.length; i++) {
+        const part = nextParts[i];
+        if (isContextSizeText(part)) {
+            contextSize = part.toUpperCase();
+            nextParts.splice(i, 1);
+            i--;
+        }
+        else if (isVersionText(part)) {
+            version = part.toLowerCase();
+            nextParts.splice(i, 1);
+            i--;
+        }
+    }
+    return {
+        name,
+        subType: previousParts.join("-"),
+        quantization,
+        fileType,
+        version,
+        contextSize,
+        parameters
+    };
+}
+function isParametersText(text) {
+    return /^[0-9]+[Bb]$/.test(text);
+}
+function isVersionText(text) {
+    return /^[vV]?[0-9]/.test(text);
+}
+function isContextSizeText(text) {
+    return /^[0-9]+[kKmM]$/.test(text);
+}
+function splitByModelParameters(parts) {
+    for (let i = 0; i < parts.length; i++) {
+        const part = parts[i];
+        if (isParametersText(part)) {
+            return {
+                parameters: part.toUpperCase(),
+                previousParts: parts.slice(0, i),
+                nextParts: parts.slice(i + 1)
+            };
+        }
+    }
+    return {
+        parameters: undefined,
+        previousParts: parts,
+        nextParts: []
+    };
+}
+//# sourceMappingURL=parseModelFileName.js.map