@realtimex/node-llama-cpp 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Gilad S.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,108 @@
+<div align="center">
+    <a href="https://node-llama-cpp.withcat.ai" target="_blank"><img alt="node-llama-cpp Logo" src="https://raw.githubusercontent.com/withcatai/node-llama-cpp/master/assets/logo.v3.roundEdges.avif" width="360px" /></a>
+    <h1>node-llama-cpp</h1>
+    <p>Run AI models locally on your machine</p>
+    <sub>Pre-built bindings are provided with a fallback to building from source with cmake</sub>
+    <p></p>
+</div>
+
+<div align="center" class="main-badges">
+
+[![Build](https://github.com/withcatai/node-llama-cpp/actions/workflows/build.yml/badge.svg)](https://github.com/withcatai/node-llama-cpp/actions/workflows/build.yml)
+[![License](https://badgen.net/badge/color/MIT/green?label=license)](https://www.npmjs.com/package/node-llama-cpp)
+[![Types](https://badgen.net/badge/color/TypeScript/blue?label=types)](https://www.npmjs.com/package/node-llama-cpp)
+[![Version](https://badgen.net/npm/v/node-llama-cpp)](https://www.npmjs.com/package/node-llama-cpp)
+
+</div>
+
+✨ [`gpt-oss` is here!](https://node-llama-cpp.withcat.ai/blog/v3.12-gpt-oss) ✨
+
+## Features
+* Run LLMs locally on your machine
+* [Metal, CUDA and Vulkan support](https://node-llama-cpp.withcat.ai/guide/#gpu-support)
+* [Pre-built binaries are provided](https://node-llama-cpp.withcat.ai/guide/building-from-source), with a fallback to building from source _**without**_ `node-gyp` or Python
+* [Adapts to your hardware automatically](https://node-llama-cpp.withcat.ai/guide/#gpu-support), no need to configure anything
+* A Complete suite of everything you need to use LLMs in your projects
+* [Use the CLI to chat with a model without writing any code](#try-it-without-installing)
+* Up-to-date with the latest `llama.cpp`. Download and compile the latest release with a [single CLI command](https://node-llama-cpp.withcat.ai/guide/building-from-source#downloading-a-release)
+* Enforce a model to generate output in a parseable format, [like JSON](https://node-llama-cpp.withcat.ai/guide/chat-session#json-response), or even force it to [follow a specific JSON schema](https://node-llama-cpp.withcat.ai/guide/chat-session#response-json-schema)
+* [Provide a model with functions it can call on demand](https://node-llama-cpp.withcat.ai/guide/chat-session#function-calling) to retrieve information or perform actions
+* [Embedding and reranking support](https://node-llama-cpp.withcat.ai/guide/embedding)
+* [Safe against special token injection attacks](https://node-llama-cpp.withcat.ai/guide/llama-text#input-safety-in-node-llama-cpp)
+* Great developer experience with full TypeScript support, and [complete documentation](https://node-llama-cpp.withcat.ai/guide/)
+* Much more
+
+## [Documentation](https://node-llama-cpp.withcat.ai)
+* [Getting started guide](https://node-llama-cpp.withcat.ai/guide/)
+* [API reference](https://node-llama-cpp.withcat.ai/api/functions/getLlama)
+* [CLI help](https://node-llama-cpp.withcat.ai/cli/)
+* [Blog](https://node-llama-cpp.withcat.ai/blog/)
+* [Changelog](https://github.com/withcatai/node-llama-cpp/releases)
+* [Roadmap](https://github.com/orgs/withcatai/projects/1)
+
+## Try It Without Installing
+Chat with a model in your terminal using [a single command](https://node-llama-cpp.withcat.ai/cli/chat):
+```bash
+npx -y node-llama-cpp chat
+```
+
+## Installation
+```bash
+npm install node-llama-cpp
+```
+
+[This package comes with pre-built binaries](https://node-llama-cpp.withcat.ai/guide/building-from-source) for macOS, Linux and Windows.
+
+If binaries are not available for your platform, it'll fallback to download a release of `llama.cpp` and build it from source with `cmake`.
+To disable this behavior, set the environment variable `NODE_LLAMA_CPP_SKIP_DOWNLOAD` to `true`.
+
+## Usage
+```typescript
+import {fileURLToPath} from "url";
+import path from "path";
+import {getLlama, LlamaChatSession} from "node-llama-cpp";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const llama = await getLlama();
+const model = await llama.loadModel({
+    modelPath: path.join(__dirname, "models", "Meta-Llama-3.1-8B-Instruct.Q4_K_M.gguf")
+});
+const context = await model.createContext();
+const session = new LlamaChatSession({
+    contextSequence: context.getSequence()
+});
+
+
+const q1 = "Hi there, how are you?";
+console.log("User: " + q1);
+
+const a1 = await session.prompt(q1);
+console.log("AI: " + a1);
+
+
+const q2 = "Summarize what you said";
+console.log("User: " + q2);
+
+const a2 = await session.prompt(q2);
+console.log("AI: " + a2);
+```
+
+> For more examples, see the [getting started guide](https://node-llama-cpp.withcat.ai/guide/)
+
+## Contributing
+To contribute to `node-llama-cpp` read the [contribution guide](https://node-llama-cpp.withcat.ai/guide/contributing).
+
+## Acknowledgements
+* llama.cpp: [ggml-org/llama.cpp](https://github.com/ggml-org/llama.cpp)
+
+
+<br />
+
+<div align="center" width="360">
+    <img alt="Star please" src="https://raw.githubusercontent.com/withcatai/node-llama-cpp/master/assets/star.please.roundEdges.png" width="360" margin="auto" />
+    <br/>
+    <p align="right">
+        <i>If you like this repo, star it ✨</i>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+    </p>
+</div>

package/dist/ChatWrapper.d.ts

@@ -0,0 +1,27 @@
+import { ChatHistoryItem, ChatModelFunctionCall, ChatModelFunctions, ChatModelResponse, ChatWrapperGenerateContextStateOptions, ChatWrapperGeneratedContextState, ChatWrapperGenerateInitialHistoryOptions, ChatWrapperSettings } from "./types.js";
+import { LlamaText } from "./utils/LlamaText.js";
+import type { JinjaTemplateChatWrapperOptions } from "./chatWrappers/generic/JinjaTemplateChatWrapper.js";
+export declare abstract class ChatWrapper {
+    static defaultSettings: ChatWrapperSettings;
+    abstract readonly wrapperName: string;
+    readonly settings: ChatWrapperSettings;
+    generateContextState({ chatHistory, availableFunctions, documentFunctionParams }: ChatWrapperGenerateContextStateOptions): ChatWrapperGeneratedContextState;
+    generateFunctionCallsAndResults(functionCalls: ChatModelFunctionCall[], useRawCall?: boolean): import("./utils/LlamaText.js")._LlamaText;
+    generateFunctionCall(name: string, params: any): LlamaText;
+    generateFunctionCallResult(functionName: string, functionParams: any, result: any): LlamaText;
+    generateModelResponseText(modelResponse: ChatModelResponse["response"], useRawValues?: boolean): LlamaText;
+    generateAvailableFunctionsSystemText(availableFunctions: ChatModelFunctions, { documentParams }: {
+        documentParams?: boolean;
+    }): LlamaText;
+    addAvailableFunctionsSystemMessageToHistory(history: readonly ChatHistoryItem[], availableFunctions?: ChatModelFunctions, { documentParams }?: {
+        documentParams?: boolean;
+    }): readonly ChatHistoryItem[];
+    generateInitialChatHistory({ systemPrompt }?: ChatWrapperGenerateInitialHistoryOptions): ChatHistoryItem[];
+}
+type FirstItemOfTupleOrFallback<T extends any[], Fallback> = T extends [infer U, ...any[]] ? U : Fallback;
+export type ChatWrapperJinjaMatchConfiguration<T extends typeof ChatWrapper> = Array<FirstItemOfTupleOrFallback<ConstructorParameters<T>, object> | [
+    testConfig: FirstItemOfTupleOrFallback<ConstructorParameters<T>, object>,
+    applyConfig: FirstItemOfTupleOrFallback<ConstructorParameters<T>, object>,
+    testJinjaChatWrapperOptions?: JinjaTemplateChatWrapperOptions
+]>;
+export {};

package/dist/ChatWrapper.js

@@ -0,0 +1,233 @@
+import { isChatModelResponseSegment } from "./types.js";
+import { LlamaText, SpecialTokensText } from "./utils/LlamaText.js";
+import { ChatModelFunctionsDocumentationGenerator } from "./chatWrappers/utils/ChatModelFunctionsDocumentationGenerator.js";
+import { jsonDumps } from "./chatWrappers/utils/jsonDumps.js";
+import { defaultChatSystemPrompt } from "./config.js";
+import { getChatWrapperSegmentDefinition } from "./utils/getChatWrapperSegmentDefinition.js";
+export class ChatWrapper {
+    static defaultSettings = {
+        supportsSystemMessages: true,
+        functions: {
+            call: {
+                optionalPrefixSpace: true,
+                prefix: "||call: ",
+                paramsPrefix: LlamaText(new SpecialTokensText("(")),
+                suffix: LlamaText(new SpecialTokensText(")")),
+                emptyCallParamsPlaceholder: ""
+            },
+            result: {
+                prefix: LlamaText(new SpecialTokensText("\n"), "||result: "),
+                suffix: LlamaText(new SpecialTokensText("\n"))
+            }
+        },
+        segments: {}
+    };
+    settings = ChatWrapper.defaultSettings;
+    generateContextState({ chatHistory, availableFunctions, documentFunctionParams }) {
+        const historyWithFunctions = this.addAvailableFunctionsSystemMessageToHistory(chatHistory, availableFunctions, {
+            documentParams: documentFunctionParams
+        });
+        const texts = historyWithFunctions
+            .map((item) => {
+            if (item.type === "system")
+                return LlamaText(["system: ", LlamaText.fromJSON(item.text)]);
+            else if (item.type === "user")
+                return LlamaText(["user: ", item.text]);
+            else if (item.type === "model")
+                return LlamaText(["model: ", this.generateModelResponseText(item.response)]);
+            return item;
+        });
+        return {
+            contextText: LlamaText.joinValues("\n", texts),
+            stopGenerationTriggers: []
+        };
+    }
+    generateFunctionCallsAndResults(functionCalls, useRawCall = true) {
+        const calls = [];
+        const results = [];
+        const res = [];
+        if (functionCalls.length === 0)
+            return LlamaText([]);
+        for (const functionCall of functionCalls) {
+            if (useRawCall && functionCall.rawCall != null)
+                calls.push(LlamaText.fromJSON(functionCall.rawCall));
+            else
+                calls.push(this.generateFunctionCall(functionCall.name, functionCall.params));
+            results.push(this.generateFunctionCallResult(functionCall.name, functionCall.params, functionCall.result));
+        }
+        if (this.settings.functions.parallelism == null) {
+            for (let i = 0; i < calls.length; i++) {
+                res.push(calls[i]);
+                res.push(results[i]);
+            }
+            return LlamaText(res);
+        }
+        res.push(LlamaText(this.settings.functions.parallelism.call.sectionPrefix ?? ""));
+        for (let i = 0; i < calls.length; i++) {
+            if (i > 0)
+                res.push(LlamaText(this.settings.functions.parallelism.call.betweenCalls ?? ""));
+            res.push(calls[i]);
+        }
+        res.push(LlamaText(this.settings.functions.parallelism.call.sectionSuffix ?? ""));
+        res.push(LlamaText(this.settings.functions.parallelism.result?.sectionPrefix ?? ""));
+        for (let i = 0; i < results.length; i++) {
+            if (i > 0)
+                res.push(LlamaText(this.settings.functions.parallelism.result?.betweenResults ?? ""));
+            res.push(results[i]);
+        }
+        res.push(LlamaText(this.settings.functions.parallelism.result?.sectionSuffix ?? ""));
+        return LlamaText(res);
+    }
+    generateFunctionCall(name, params) {
+        const emptyCallParamsPlaceholder = this.settings.functions.call.emptyCallParamsPlaceholder;
+        return LlamaText([
+            this.settings.functions.call.prefix,
+            name,
+            this.settings.functions.call.paramsPrefix,
+            (params === undefined
+                ? (emptyCallParamsPlaceholder === undefined || emptyCallParamsPlaceholder === "")
+                    ? ""
+                    : jsonDumps(emptyCallParamsPlaceholder)
+                : jsonDumps(params)),
+            this.settings.functions.call.suffix
+        ]);
+    }
+    generateFunctionCallResult(functionName, functionParams, result) {
+        return this._generateFunctionCallResult(functionName, functionParams, result === undefined
+            ? "void"
+            : jsonDumps(result));
+    }
+    /** @internal */
+    _generateFunctionCallResult(functionName, functionParams, rawResult) {
+        function resolveParameters(text) {
+            return LlamaText(text)
+                .mapValues((value) => {
+                if (typeof value !== "string")
+                    return value;
+                return value
+                    .replaceAll("{{functionName}}", functionName)
+                    .replaceAll("{{functionParams}}", functionParams === undefined ? "" : jsonDumps(functionParams));
+            });
+        }
+        return LlamaText([
+            resolveParameters(this.settings.functions.result.prefix),
+            rawResult,
+            resolveParameters(this.settings.functions.result.suffix)
+        ]);
+    }
+    generateModelResponseText(modelResponse, useRawValues = true) {
+        const res = [];
+        const pendingFunctionCalls = [];
+        const segmentStack = [];
+        let lastSegmentEndedWithoutSuffix = false;
+        let needsToAddSegmentReminder = false;
+        const addFunctionCalls = () => {
+            if (pendingFunctionCalls.length === 0)
+                return;
+            res.push(this.generateFunctionCallsAndResults(pendingFunctionCalls, useRawValues));
+            pendingFunctionCalls.length = 0;
+            needsToAddSegmentReminder = true;
+        };
+        const addSegmentReminderIfNeeded = () => {
+            if (lastSegmentEndedWithoutSuffix && segmentStack.length === 0 && this.settings.segments?.closeAllSegments != null) {
+                lastSegmentEndedWithoutSuffix = false;
+                res.push(LlamaText(this.settings.segments.closeAllSegments));
+            }
+            else if (needsToAddSegmentReminder && segmentStack.length > 0 && this.settings.segments?.reiterateStackAfterFunctionCalls) {
+                for (const segmentType of segmentStack) {
+                    const segmentDefinition = getChatWrapperSegmentDefinition(this.settings, segmentType);
+                    if (segmentDefinition == null)
+                        continue;
+                    res.push(LlamaText(segmentDefinition.prefix));
+                }
+            }
+        };
+        for (const response of modelResponse) {
+            if (typeof response === "string") {
+                addFunctionCalls();
+                addSegmentReminderIfNeeded();
+                res.push(LlamaText(response));
+                continue;
+            }
+            else if (isChatModelResponseSegment(response)) {
+                addFunctionCalls();
+                const segmentDefinition = getChatWrapperSegmentDefinition(this.settings, response.segmentType);
+                if (response.raw != null && useRawValues)
+                    res.push(LlamaText.fromJSON(response.raw));
+                else
+                    res.push(LlamaText([
+                        (segmentStack.length > 0 && segmentStack.at(-1) === response.segmentType)
+                            ? ""
+                            : segmentDefinition?.prefix ?? "",
+                        response.text,
+                        response.ended
+                            ? (segmentDefinition?.suffix ?? "")
+                            : ""
+                    ]));
+                lastSegmentEndedWithoutSuffix = response.ended && segmentDefinition?.suffix == null;
+                if (!response.ended && segmentStack.at(-1) !== response.segmentType)
+                    segmentStack.push(response.segmentType);
+                else if (response.ended && segmentStack.at(-1) === response.segmentType) {
+                    segmentStack.pop();
+                    if (segmentStack.length === 0 && segmentDefinition?.suffix == null && this.settings.segments?.closeAllSegments != null)
+                        res.push(LlamaText(this.settings.segments.closeAllSegments));
+                }
+                continue;
+            }
+            if (response.startsNewChunk)
+                addFunctionCalls();
+            pendingFunctionCalls.push(response);
+        }
+        addFunctionCalls();
+        addSegmentReminderIfNeeded();
+        return LlamaText(res);
+    }
+    generateAvailableFunctionsSystemText(availableFunctions, { documentParams = true }) {
+        const functionsDocumentationGenerator = new ChatModelFunctionsDocumentationGenerator(availableFunctions);
+        if (!functionsDocumentationGenerator.hasAnyFunctions)
+            return LlamaText([]);
+        return LlamaText.joinValues("\n", [
+            "The assistant calls the provided functions as needed to retrieve information instead of relying on existing knowledge.",
+            "To fulfill a request, the assistant calls relevant functions in advance when needed before responding to the request, and does not tell the user prior to calling a function.",
+            "Provided functions:",
+            "```typescript",
+            functionsDocumentationGenerator.getTypeScriptFunctionSignatures({ documentParams }),
+            "```",
+            "",
+            "Calling any of the provided functions can be done like this:",
+            this.generateFunctionCall("getSomeInfo", { someKey: "someValue" }),
+            "",
+            "Note that the || prefix is mandatory.",
+            "The assistant does not inform the user about using functions and does not explain anything before calling a function.",
+            "After calling a function, the raw result appears afterwards and is not part of the conversation.",
+            "To make information be part of the conversation, the assistant paraphrases and repeats the information without the function syntax."
+        ]);
+    }
+    addAvailableFunctionsSystemMessageToHistory(history, availableFunctions, { documentParams = true } = {}) {
+        const availableFunctionNames = Object.keys(availableFunctions ?? {});
+        if (availableFunctions == null || availableFunctionNames.length === 0)
+            return history;
+        const res = history.slice();
+        const firstNonSystemMessageIndex = res.findIndex((item) => item.type !== "system");
+        res.splice(Math.max(0, firstNonSystemMessageIndex), 0, {
+            type: "system",
+            text: this.generateAvailableFunctionsSystemText(availableFunctions, { documentParams }).toJSON()
+        });
+        return res;
+    }
+    generateInitialChatHistory({ systemPrompt = defaultChatSystemPrompt } = {}) {
+        return [{
+                type: "system",
+                text: LlamaText(systemPrompt ?? defaultChatSystemPrompt).toJSON()
+            }];
+    }
+    /** @internal */
+    static _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate() {
+        return [{}];
+    }
+    /** @internal */
+    static _checkModelCompatibility(options) {
+        return true;
+    }
+}
+//# sourceMappingURL=ChatWrapper.js.map

package/dist/ChatWrapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChatWrapper.js","sourceRoot":"","sources":["../src/ChatWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAG4D,0BAA0B,EAC5F,MAAM,YAAY,CAAC;AACpB,OAAO,EAAC,SAAS,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAC,wCAAwC,EAAC,MAAM,kEAAkE,CAAC;AAC1H,OAAO,EAAC,SAAS,EAAC,MAAM,mCAAmC,CAAC;AAC5D,OAAO,EAAC,uBAAuB,EAAC,MAAM,aAAa,CAAC;AACpD,OAAO,EAAC,+BAA+B,EAAC,MAAM,4CAA4C,CAAC;AAG3F,MAAM,OAAgB,WAAW;IACtB,MAAM,CAAC,eAAe,GAAwB;QACjD,sBAAsB,EAAE,IAAI;QAC5B,SAAS,EAAE;YACP,IAAI,EAAE;gBACF,mBAAmB,EAAE,IAAI;gBACzB,MAAM,EAAE,UAAU;gBAClB,YAAY,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBACnD,MAAM,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBAC7C,0BAA0B,EAAE,EAAE;aACjC;YACD,MAAM,EAAE;gBACJ,MAAM,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;gBAC5D,MAAM,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC;aACjD;SACJ;QACD,QAAQ,EAAE,EAAE;KACf,CAAC;IAGc,QAAQ,GAAwB,WAAW,CAAC,eAAe,CAAC;IAErE,oBAAoB,CAAC,EACxB,WAAW,EAAE,kBAAkB,EAAE,sBAAsB,EAClB;QACrC,MAAM,oBAAoB,GAAG,IAAI,CAAC,2CAA2C,CAAC,WAAW,EAAE,kBAAkB,EAAE;YAC3G,cAAc,EAAE,sBAAsB;SACzC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,oBAAoB;aAC7B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;gBACtB,OAAO,SAAS,CAAC,CAAC,UAAU,EAAE,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;iBAC7D,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;gBACzB,OAAO,SAAS,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;iBACvC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;gBAC1B,OAAO,SAAS,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAEjF,OAAO,IAAoB,CAAC;QAChC,CAAC,CAAC,CAAC;QAEP,OAAO;YACH,WAAW,EAAE,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC;YAC9C,sBAAsB,EAAE,EAAE;SAC7B,CAAC;IACN,CAAC;IAEM,+BAA+B,CAAC,aAAsC,EAAE,aAAsB,IAAI;QACrG,MAAM,KAAK,GAAgB,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,MAAM,GAAG,GAAgB,EAAE,CAAC;QAE5B,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAC1B,OAAO,SAAS,CAAC,EAAE,CAAC,CAAC;QAEzB,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;YACvC,IAAI,UAAU,IAAI,YAAY,CAAC,OAAO,IAAI,IAAI;gBAC1C,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;;gBAErD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;YAElF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/G,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;YAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;gBACpB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;YAC1B,CAAC;YAED,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC;QAClF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,CAAC;gBACL,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;YAErF,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;QACxB,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC;QAElF,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC;QACrF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,GAAG,CAAC;gBACL,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,cAAc,IAAI,EAAE,CAAC,CAAC,CAAC;YAE1F,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;QAC1B,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC;QAErF,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEM,oBAAoB,CAAC,IAAY,EAAE,MAAW;QACjD,MAAM,0BAA0B,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,0BAA0B,CAAC;QAC3F,OAAO,SAAS,CAAC;YACb,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM;YACnC,IAAI;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY;YACzC,CACI,MAAM,KAAK,SAAS;gBAChB,CAAC,CAAC,CAAC,0BAA0B,KAAK,SAAS,IAAI,0BAA0B,KAAK,EAAE,CAAC;oBAC7E,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC,SAAS,CAAC,0BAA0B,CAAC;gBAC3C,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAC1B;YACD,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM;SACtC,CAAC,CAAC;IACP,CAAC;IAEM,0BAA0B,CAAC,YAAoB,EAAE,cAAmB,EAAE,MAAW;QACpF,OAAO,IAAI,CAAC,2BAA2B,CACnC,YAAY,EACZ,cAAc,EACd,MAAM,KAAK,SAAS;YAChB,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAC1B,CAAC;IACN,CAAC;IAED,gBAAgB;IACN,2BAA2B,CAAC,YAAoB,EAAE,cAAmB,EAAE,SAAiB;QAC9F,SAAS,iBAAiB,CAAC,IAAwB;YAC/C,OAAO,SAAS,CAAC,IAAI,CAAC;iBACjB,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE;gBACjB,IAAI,OAAO,KAAK,KAAK,QAAQ;oBACzB,OAAO,KAAK,CAAC;gBAEjB,OAAO,KAAK;qBACP,UAAU,CAAC,kBAAkB,EAAE,YAAY,CAAC;qBAC5C,UAAU,CAAC,oBAAoB,EAAE,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;YACzG,CAAC,CAAC,CAAC;QACX,CAAC;QAED,OAAO,SAAS,CAAC;YACb,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;YACxD,SAAS;YACT,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;SAC3D,CAAC,CAAC;IACP,CAAC;IAEM,yBAAyB,CAAC,aAA4C,EAAE,eAAwB,IAAI;QACvG,MAAM,GAAG,GAAgB,EAAE,CAAC;QAC5B,MAAM,oBAAoB,GAA4B,EAAE,CAAC;QACzD,MAAM,YAAY,GAA2B,EAAE,CAAC;QAChD,IAAI,6BAA6B,GAAY,KAAK,CAAC;QACnD,IAAI,yBAAyB,GAAG,KAAK,CAAC;QAEtC,MAAM,gBAAgB,GAAG,GAAG,EAAE;YAC1B,IAAI,oBAAoB,CAAC,MAAM,KAAK,CAAC;gBACjC,OAAO;YAEX,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CAAC,oBAAoB,EAAE,YAAY,CAAC,CAAC,CAAC;YACnF,oBAAoB,CAAC,MAAM,GAAG,CAAC,CAAC;YAChC,yBAAyB,GAAG,IAAI,CAAC;QACrC,CAAC,CAAC;QAEF,MAAM,0BAA0B,GAAG,GAAG,EAAE;YACpC,IAAI,6BAA6B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,IAAI,IAAI,EAAE,CAAC;gBACjH,6BAA6B,GAAG,KAAK,CAAC;gBACtC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC;YACjE,CAAC;iBAAM,IAAI,yBAAyB,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,gCAAgC,EAAE,CAAC;gBAC1H,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;oBACrC,MAAM,iBAAiB,GAAG,+BAA+B,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;oBACtF,IAAI,iBAAiB,IAAI,IAAI;wBACzB,SAAS;oBAEb,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;gBAClD,CAAC;YACL,CAAC;QACL,CAAC,CAAC;QAEF,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;YACnC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC/B,gBAAgB,EAAE,CAAC;gBACnB,0BAA0B,EAAE,CAAC;gBAE7B,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC9B,SAAS;YACb,CAAC;iBAAM,IAAI,0BAA0B,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9C,gBAAgB,EAAE,CAAC;gBAEnB,MAAM,iBAAiB,GAAG,+BAA+B,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC/F,IAAI,QAAQ,CAAC,GAAG,IAAI,IAAI,IAAI,YAAY;oBACpC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;;oBAE3C,GAAG,CAAC,IAAI,CACJ,SAAS,CAAC;wBACN,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,WAAW,CAAC;4BACrE,CAAC,CAAC,EAAE;4BACJ,CAAC,CAAC,iBAAiB,EAAE,MAAM,IAAI,EAAE;wBACrC,QAAQ,CAAC,IAAI;wBACb,QAAQ,CAAC,KAAK;4BACV,CAAC,CAAC,CAAC,iBAAiB,EAAE,MAAM,IAAI,EAAE,CAAC;4BACnC,CAAC,CAAC,EAAE;qBACX,CAAC,CACL,CAAC;gBAEN,6BAA6B,GAAG,QAAQ,CAAC,KAAK,IAAI,iBAAiB,EAAE,MAAM,IAAI,IAAI,CAAC;gBAEpF,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,WAAW;oBAC/D,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;qBACvC,IAAI,QAAQ,CAAC,KAAK,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,WAAW,EAAE,CAAC;oBACtE,YAAY,CAAC,GAAG,EAAE,CAAC;oBAEnB,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,iBAAiB,EAAE,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,IAAI,IAAI;wBAClH,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBACrE,CAAC;gBAED,SAAS;YACb,CAAC;YAED,IAAI,QAAQ,CAAC,cAAc;gBACvB,gBAAgB,EAAE,CAAC;YAEvB,oBAAoB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,CAAC;QAED,gBAAgB,EAAE,CAAC;QACnB,0BAA0B,EAAE,CAAC;QAE7B,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEM,oCAAoC,CAAC,kBAAsC,EAAE,EAAC,cAAc,GAAG,IAAI,EAEzG;QACG,MAAM,+BAA+B,GAAG,IAAI,wCAAwC,CAAC,kBAAkB,CAAC,CAAC;QAEzG,IAAI,CAAC,+BAA+B,CAAC,eAAe;YAChD,OAAO,SAAS,CAAC,EAAE,CAAC,CAAC;QAEzB,OAAO,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE;YAC9B,wHAAwH;YACxH,+KAA+K;YAC/K,qBAAqB;YACrB,eAAe;YACf,+BAA+B,CAAC,+BAA+B,CAAC,EAAC,cAAc,EAAC,CAAC;YACjF,KAAK;YACL,EAAE;YACF,8DAA8D;YAC9D,IAAI,CAAC,oBAAoB,CAAC,aAAa,EAAE,EAAC,OAAO,EAAE,WAAW,EAAC,CAAC;YAChE,EAAE;YACF,uCAAuC;YACvC,uHAAuH;YACvH,kGAAkG;YAClG,qIAAqI;SACxI,CAAC,CAAC;IACP,CAAC;IAEM,2CAA2C,CAAC,OAAmC,EAAE,kBAAuC,EAAE,EAC7H,cAAc,GAAG,IAAI,KAGrB,EAAE;QACF,MAAM,sBAAsB,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;QAErE,IAAI,kBAAkB,IAAI,IAAI,IAAI,sBAAsB,CAAC,MAAM,KAAK,CAAC;YACjE,OAAO,OAAO,CAAC;QAEnB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAE5B,MAAM,0BAA0B,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACnF,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,0BAA0B,CAAC,EAAE,CAAC,EAAE;YACnD,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,IAAI,CAAC,oCAAoC,CAAC,kBAAkB,EAAE,EAAC,cAAc,EAAC,CAAC,CAAC,MAAM,EAAE;SACjG,CAAC,CAAC;QAEH,OAAO,GAAG,CAAC;IACf,CAAC;IAEM,0BAA0B,CAAC,EAC9B,YAAY,GAAG,uBAAuB,KACI,EAAE;QAC5C,OAAO,CAAC;gBACJ,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,SAAS,CAAC,YAAY,IAAI,uBAAuB,CAAC,CAAC,MAAM,EAAE;aACpE,CAAC,CAAC;IACP,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,yDAAyD;QACnE,OAAO,CAAC,EAAE,CAA2D,CAAC;IAC1E,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,wBAAwB,CAAC,OAAiD;QACpF,OAAO,IAAI,CAAC;IAChB,CAAC"}

package/dist/apiDocsIndex.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/apiDocsIndex.js

@@ -0,0 +1,7 @@
+/** @internal */
+import { _LlamaText } from "./utils/LlamaText.js";
+/** @internal */
+export * from "./index.js";
+/** @internal */
+export { _LlamaText as LlamaText };
+//# sourceMappingURL=apiDocsIndex.js.map

package/dist/apiDocsIndex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"apiDocsIndex.js","sourceRoot":"","sources":["../src/apiDocsIndex.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,OAAO,EACH,UAAU,EACb,MAAM,sBAAsB,CAAC;AAE9B,gBAAgB;AAChB,cAAc,YAAY,CAAC;AAE3B,gBAAgB;AAChB,OAAO,EACH,UAAU,IAAI,SAAS,EAC1B,CAAC"}

package/dist/bindings/AddonTypes.d.ts

@@ -0,0 +1,203 @@
+import { Token } from "../types.js";
+import { LlamaNuma } from "./types.js";
+export type BindingModule = {
+    AddonModel: {
+        new (modelPath: string, params: {
+            addonExports?: BindingModule;
+            gpuLayers?: number;
+            vocabOnly?: boolean;
+            useMmap?: boolean;
+            useDirectIo?: boolean;
+            useMlock?: boolean;
+            checkTensors?: boolean;
+            onLoadProgress?(loadPercentage: number): void;
+            hasLoadAbortSignal?: boolean;
+            overridesList?: Array<[key: string, value: number | bigint | boolean | string, type: 0 | 1 | undefined]>;
+        }): AddonModel;
+    };
+    AddonModelLora: {
+        new (model: AddonModel, filePath: string): AddonModelLora;
+    };
+    AddonContext: {
+        new (model: AddonModel, params: {
+            contextSize?: number;
+            batchSize?: number;
+            sequences?: number;
+            flashAttention?: boolean;
+            logitsAll?: boolean;
+            embeddings?: boolean;
+            ranking?: boolean;
+            threads?: number;
+            performanceTracking?: boolean;
+            kvCacheKeyType?: number;
+            kvCacheValueType?: number;
+            swaFullCache?: boolean;
+        }): AddonContext;
+    };
+    AddonContextSequenceCheckpoint: {
+        new (): AddonContextSequenceCheckpoint;
+    };
+    AddonGrammar: {
+        new (grammarPath: string, params?: {
+            addonExports?: BindingModule;
+            rootRuleName?: string;
+        }): AddonGrammar;
+    };
+    AddonGrammarEvaluationState: {
+        new (model: AddonModel, grammar: AddonGrammar): AddonGrammarEvaluationState;
+        new (existingState: AddonGrammarEvaluationState): AddonGrammarEvaluationState;
+    };
+    AddonSampler: {
+        new (model: AddonModel): AddonSampler;
+        acceptGrammarEvaluationStateToken(grammarEvaluationState: AddonGrammarEvaluationState, token: Token): void;
+        canBeNextTokenForGrammarEvaluationState(grammarEvaluationState: AddonGrammarEvaluationState, token: Token): boolean;
+    };
+    markLoaded(): boolean;
+    systemInfo(): string;
+    getSupportsGpuOffloading(): boolean;
+    getSupportsMmap(): boolean;
+    getGpuSupportsMmap(): boolean;
+    getSupportsMlock(): boolean;
+    getMathCores(): number;
+    getBlockSizeForGgmlType(ggmlType: number): number | undefined;
+    getTypeSizeForGgmlType(ggmlType: number): number | undefined;
+    getGgmlGraphOverheadCustom(size: number, grads: boolean): number;
+    getConsts(): {
+        ggmlMaxDims: number;
+        ggmlTypeF16Size: number;
+        ggmlTypeF32Size: number;
+        ggmlTensorOverhead: number;
+        llamaPosSize: number;
+        llamaSeqIdSize: number;
+    };
+    setLogger(logger: (level: number, message: string) => void): void;
+    setLoggerLogLevel(level: number): void;
+    getGpuVramInfo(): {
+        total: number;
+        used: number;
+        unifiedSize: number;
+    };
+    getGpuDeviceInfo(): {
+        deviceNames: string[];
+    };
+    getGpuType(): "cuda" | "vulkan" | "metal" | false | undefined;
+    ensureGpuDeviceIsSupported(): void;
+    getSwapInfo(): {
+        total: number;
+        maxSize: number;
+        free: number;
+    };
+    getMemoryInfo(): {
+        total: number;
+    };
+    init(): Promise<void>;
+    setNuma(numa?: LlamaNuma): void;
+    loadBackends(forceLoadLibrariesSearchPath?: string): void;
+    dispose(): Promise<void>;
+};
+export type AddonModel = {
+    init(): Promise<boolean>;
+    loadLora(lora: AddonModelLora): Promise<void>;
+    abortActiveModelLoad(): void;
+    dispose(): Promise<void>;
+    tokenize(text: string, specialTokens: boolean): Uint32Array;
+    detokenize(tokens: Uint32Array, specialTokens?: boolean): string;
+    getTrainContextSize(): number;
+    getEmbeddingVectorSize(): number;
+    getTotalSize(): number;
+    getTotalParameters(): number;
+    getModelDescription(): ModelTypeDescription;
+    tokenBos(): Token;
+    tokenEos(): Token;
+    tokenNl(): Token;
+    prefixToken(): Token;
+    middleToken(): Token;
+    suffixToken(): Token;
+    eotToken(): Token;
+    sepToken(): Token;
+    getTokenString(token: number): string;
+    getTokenAttributes(token: Token): number;
+    isEogToken(token: Token): boolean;
+    getVocabularyType(): number;
+    shouldPrependBosToken(): boolean;
+    shouldAppendEosToken(): boolean;
+    getModelSize(): number;
+};
+export type AddonContext = {
+    init(): Promise<boolean>;
+    dispose(): Promise<void>;
+    getContextSize(): number;
+    initBatch(size: number): void;
+    addToBatch(sequenceId: number, firstTokenSequenceIndex: number, tokens: Uint32Array, logitIndexes: Uint32Array): Uint32Array;
+    decodeBatch(): Promise<void>;
+    sampleToken(batchLogitIndex: BatchLogitIndex, sampler: AddonSampler): Promise<Token | -1>;
+    sampleToken(batchLogitIndex: BatchLogitIndex, sampler: AddonSampler, probabilities: boolean, confidence?: boolean): Promise<[token: Token | -1, probabilities: (Token | number)[] | undefined, confidence: number | undefined]>;
+    disposeSequence(sequenceId: number): void;
+    removeTokenCellsFromSequence(sequenceId: number, startPos: number, endPos: number): boolean;
+    shiftSequenceTokenCells(sequenceId: number, startPos: number, endPos: number, shiftDelta: number): void;
+    getSequenceKvCacheMinPosition(sequenceId: number): number;
+    getSequenceKvCacheMaxPosition(sequenceId: number): number;
+    getEmbedding(inputTokensLength: number, maxVectorSize?: number): Float64Array;
+    getStateSize(): number;
+    getThreads(): number;
+    setThreads(threads: number): void;
+    printTimings(): void;
+    ensureDraftContextIsCompatibleForSpeculative(draftContext: AddonContext): void;
+    saveSequenceStateToFile(filePath: string, sequenceId: number, tokens: Uint32Array): Promise<number>;
+    loadSequenceStateFromFile(filePath: string, sequenceId: number, maxContextSize: number): Promise<Uint32Array>;
+    setLoras(loras: AddonModelLora[], scales: number[]): void;
+    restoreCheckpoint(checkpoint: AddonContextSequenceCheckpoint, maxPosIndex: number): Promise<boolean>;
+};
+export type AddonContextSequenceCheckpoint = {
+    init(context: AddonContext, sequenceId: number): Promise<void>;
+    dispose(): void;
+    get size(): number;
+    get minPos(): number;
+    get maxPos(): number;
+};
+export type BatchLogitIndex = number & {
+    readonly __batchLogitIndex: never;
+};
+export type AddonGrammar = {
+    isTextCompatible(testText: string): boolean;
+};
+export type AddonGrammarEvaluationState = "AddonGrammarEvaluationState" & {
+    readonly __brand: never;
+};
+export type AddonSampler = {
+    dispose(): void;
+    applyConfig(config: {
+        temperature?: number;
+        minP?: number;
+        topK?: number;
+        topP?: number;
+        seed?: number;
+        xtcProbability?: number;
+        xtcThreshold?: number;
+        repeatPenalty?: number;
+        repeatPenaltyMaxTokens?: number;
+        repeatPenaltyTokens?: Uint32Array;
+        repeatPenaltyPresencePenalty?: number;
+        repeatPenaltyFrequencyPenalty?: number;
+        dryRepeatPenaltyStrength?: number;
+        dryRepeatPenaltyBase?: number;
+        dryRepeatPenaltyAllowedLength?: number;
+        dryRepeatPenaltyLastTokens?: number;
+        dryRepeatPenaltySequenceBreakers?: false | string[];
+        grammarEvaluationState?: AddonGrammarEvaluationState;
+        tokenBiasKeys?: Uint32Array;
+        tokenBiasValues?: Float32Array;
+    }): void;
+};
+export type AddonModelLora = {
+    usages: number;
+    readonly filePath: string;
+    readonly disposed: boolean;
+    dispose(): Promise<void>;
+};
+export type ModelTypeDescription = `${AddonModelArchName} ${AddonModelTypeName} ${AddonModelFileTypeName}`;
+export type AddonModelArchName = "unknown" | "llama" | "falcon" | "gpt2" | "gptj" | "gptneox" | "mpt" | "baichuan" | "starcoder" | "persimmon" | "refact" | "bloom" | "stablelm";
+export type AddonModelTypeName = "1B" | "3B" | "7B" | "8B" | "13B" | "15B" | "30B" | "34B" | "40B" | "65B" | "70B" | "?B";
+export type AddonModelFileTypeName = _AddonModelFileTypeName | `${_AddonModelFileTypeName} (guessed)`;
+type _AddonModelFileTypeName = "all F32" | "mostly F16" | "mostly Q4_0" | "mostly Q4_1" | "mostly Q4_1, some F16" | "mostly Q5_0" | "mostly Q5_1" | "mostly Q8_0" | "mostly Q2_K" | "mostly Q3_K - Small" | "mostly Q3_K - Medium" | "mostly Q3_K - Large" | "mostly Q4_K - Small" | "mostly Q4_K - Medium" | "mostly Q5_K - Small" | "mostly Q5_K - Medium" | "mostly Q6_K" | "unknown, may not work";
+export {};

package/dist/bindings/AddonTypes.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=AddonTypes.js.map

package/dist/bindings/AddonTypes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AddonTypes.js","sourceRoot":"","sources":["../../src/bindings/AddonTypes.ts"],"names":[],"mappings":""}