node-llama-cpp 3.0.0-beta.9 → 3.0.1

package/dist/types.d.ts

@@ -1,16 +1,105 @@
 import { GbnfJsonSchema, GbnfJsonSchemaToType } from "./utils/gbnfJson/types.js";
-import { BuiltinSpecialTokenValue } from "./utils/LlamaText.js";
+import { LlamaText, BuiltinSpecialTokenValue, LlamaTextJSON } from "./utils/LlamaText.js";
+import type { GgufFileInfo } from "./gguf/types/GgufFileInfoTypes.js";
 export type Token = number & {
     __token: never;
 };
+export type Detokenizer = {
+    detokenize(tokens: readonly Token[], specialTokens?: boolean, lastTokens?: readonly Token[]): string;
+}["detokenize"];
 export type Tokenizer = {
-    tokenize(text: string, specialTokens?: boolean): Token[];
+    tokenize(text: string, specialTokens?: boolean, options?: "trimLeadingSpace"): Token[];
     tokenize(text: BuiltinSpecialTokenValue, specialTokens: "builtin"): Token[];
-}["tokenize"];
+}["tokenize"] & {
+    readonly detokenize: Detokenizer;
+    isSpecialToken(token: Token): boolean;
+    isEogToken(token: Token): boolean;
+};
+export type ChatWrapperSettings = {
+    readonly supportsSystemMessages: boolean;
+    readonly functions: {
+        readonly call: {
+            readonly optionalPrefixSpace: boolean;
+            readonly prefix: string | LlamaText;
+            readonly paramsPrefix: string | LlamaText;
+            readonly suffix: string | LlamaText;
+        };
+        readonly result: {
+            /**
+             * Supported template parameters:
+             * - <span v-pre>`{{functionName}}`</span>
+             * - <span v-pre>`{{functionParams}}`</span>
+             *
+             * Template parameters can only appear in a string or a string in a `LlamaText`.
+             *
+             * Template parameters inside a `SpecialTokensText` inside a `LlamaText` won't be replaced.
+             *
+             * Example of supported values:
+             * - `"text{{functionName}}text"`
+             * - `LlamaText(["text{{functionName}}text"])`
+             *
+             * Example of unsupported values:
+             * - `LlamaText([new SpecialTokensText("text{{functionName}}text")])`
+             */
+            readonly prefix: string | LlamaText;
+            /**
+             * Supported template parameters:
+             * - <span v-pre>`{{functionName}}`</span>
+             * - <span v-pre>`{{functionParams}}`</span>
+             *
+             * Template parameters can only appear in a string or a string in a `LlamaText`.
+             *
+             * Template parameters inside a `SpecialTokensText` inside a `LlamaText` won't be replaced.
+             *
+             * Example of **supported** values:
+             * - `"text{{functionName}}text"`
+             * - `LlamaText(["text{{functionName}}text"])`
+             *
+             * Example of **unsupported** values:
+             * - `LlamaText([new SpecialTokensText("text{{functionName}}text")])`
+             */
+            readonly suffix: string | LlamaText;
+        };
+        /** If this field is present, parallel function calling is supported */
+        readonly parallelism?: {
+            readonly call: {
+                readonly sectionPrefix: string | LlamaText;
+                readonly betweenCalls?: string | LlamaText;
+                readonly sectionSuffix?: string | LlamaText;
+            };
+            readonly result?: {
+                readonly sectionPrefix?: string | LlamaText;
+                readonly betweenResults?: string | LlamaText;
+                readonly sectionSuffix?: string | LlamaText;
+            };
+        };
+    };
+};
+export type ChatWrapperGenerateContextStateOptions = {
+    chatHistory: readonly ChatHistoryItem[];
+    availableFunctions?: ChatModelFunctions;
+    documentFunctionParams?: boolean;
+};
+export type ChatWrapperCheckModelCompatibilityParams = {
+    tokenizer?: Tokenizer;
+    fileInfo?: GgufFileInfo;
+};
+export type ChatWrapperGeneratedContextState = {
+    contextText: LlamaText;
+    stopGenerationTriggers: LlamaText[];
+    ignoreStartText?: LlamaText[];
+    functionCall?: {
+        initiallyEngaged: boolean;
+        disengageInitiallyEngaged: LlamaText[];
+    };
+};
+export type ChatWrapperGenerateInitialHistoryOptions = {
+    systemPrompt?: string;
+};
 export type ChatHistoryItem = ChatSystemMessage | ChatUserMessage | ChatModelResponse;
 export type ChatSystemMessage = {
     type: "system";
-    text: string;
+    text: string | LlamaTextJSON;
 };
 export type ChatUserMessage = {
     type: "user";
@@ -26,7 +115,13 @@ export type ChatModelFunctionCall = {
     description?: string;
     params: any;
     result: any;
-    raw?: string;
+    rawCall?: LlamaTextJSON;
+    /**
+     * Whether this function call starts a new function calling chunk.
+     *
+     * Relevant only when parallel function calling is supported.
+     */
+    startsNewChunk?: boolean;
 };
 export type ChatModelFunctions = {
     readonly [name: string]: {
@@ -43,3 +138,34 @@ export type ChatSessionModelFunction<Params extends GbnfJsonSchema | undefined =
     readonly handler: (params: GbnfJsonSchemaToType<Params>) => any;
 };
 export declare function isChatModelResponseFunctionCall(item: ChatModelResponse["response"][number]): item is ChatModelFunctionCall;
+export type LLamaContextualRepeatPenalty = {
+    /**
+     * Number of recent tokens generated by the model to apply penalties to repetition of.
+     * Defaults to `64`.
+     */
+    lastTokens?: number;
+    punishTokensFilter?: (tokens: Token[]) => Token[];
+    /**
+     * Penalize new line tokens.
+     * Enabled by default.
+     */
+    penalizeNewLine?: boolean;
+    /**
+     * The relative amount to lower the probability of the tokens in `punishTokens` by
+     * Defaults to `1.1`.
+     * Set to `1` to disable.
+     */
+    penalty?: number;
+    /**
+     * For n time a token is in the `punishTokens` array, lower its probability by `n * frequencyPenalty`
+     * Disabled by default (`0`).
+     * Set to a value between `0` and `1` to enable.
+     */
+    frequencyPenalty?: number;
+    /**
+     * Lower the probability of all the tokens in the `punishTokens` array by `presencePenalty`
+     * Disabled by default (`0`).
+     * Set to a value between `0` and `1` to enable.
+     */
+    presencePenalty?: number;
+};

package/dist/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAqDA,MAAM,UAAU,+BAA+B,CAAC,IAA2C;IACvF,IAAI,OAAO,IAAI,KAAK,QAAQ;QACxB,OAAO,KAAK,CAAC;IAEjB,OAAO,IAAI,CAAC,IAAI,KAAK,cAAc,CAAC;AACxC,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AA6JA,MAAM,UAAU,+BAA+B,CAAC,IAA2C;IACvF,IAAI,OAAO,IAAI,KAAK,QAAQ;QACxB,OAAO,KAAK,CAAC;IAEjB,OAAO,IAAI,CAAC,IAAI,KAAK,cAAc,CAAC;AACxC,CAAC"}

package/dist/utils/DisposeGuard.d.ts

@@ -0,0 +1,13 @@
+export declare class DisposeGuard {
+    constructor(parentDisposeGuards?: DisposeGuard[]);
+    addParentDisposeGuard(parent: DisposeGuard): void;
+    removeParentDisposeGuard(parent: DisposeGuard): void;
+    acquireDisposeLock(): Promise<void>;
+    createPreventDisposalHandle(ignoreAwaitingDispose?: boolean): DisposalPreventionHandle;
+}
+export declare class DisposalPreventionHandle {
+    private constructor();
+    dispose(): void;
+    [Symbol.dispose](): void;
+    get disposed(): boolean;
+}

package/dist/utils/DisposeGuard.js

@@ -0,0 +1,120 @@
+import { DisposedError } from "lifecycle-utils";
+export class DisposeGuard {
+    /** @internal */ _preventionHandles = 0;
+    /** @internal */ _awaitingDisposeLockCallbacks = [];
+    /** @internal */ _disposeActivated = false;
+    /** @internal */ _parentDisposeGuardsLocks = new Map();
+    constructor(parentDisposeGuards = []) {
+        for (const parent of parentDisposeGuards)
+            this._parentDisposeGuardsLocks.set(parent, null);
+    }
+    addParentDisposeGuard(parent) {
+        if (this._parentDisposeGuardsLocks.has(parent))
+            return;
+        this._parentDisposeGuardsLocks.set(parent, null);
+        if (this._preventionHandles > 0)
+            this._parentDisposeGuardsLocks.set(parent, parent.createPreventDisposalHandle(true));
+    }
+    removeParentDisposeGuard(parent) {
+        const parentLock = this._parentDisposeGuardsLocks.get(parent);
+        if (parentLock != null) {
+            parentLock.dispose();
+            this._parentDisposeGuardsLocks.delete(parent);
+        }
+    }
+    async acquireDisposeLock() {
+        return new Promise((accept) => {
+            if (this._preventionHandles > 0)
+                this._awaitingDisposeLockCallbacks.push(accept);
+            else {
+                this._disposeActivated = true;
+                accept();
+            }
+        });
+    }
+    createPreventDisposalHandle(ignoreAwaitingDispose = false) {
+        if (this._isDisposeActivated() || (!ignoreAwaitingDispose && this._hasAwaitingDisposeLocks()))
+            throw new DisposedError();
+        this._preventionHandles++;
+        try {
+            this._updateParentDisposeGuardLocks();
+        }
+        catch (err) {
+            this._preventionHandles--;
+            if (this._preventionHandles === 0)
+                this._updateParentDisposeGuardLocks();
+            throw err;
+        }
+        return DisposalPreventionHandle._create(() => {
+            this._preventionHandles--;
+            this._activateLocksIfNeeded();
+            this._updateParentDisposeGuardLocks(true);
+        });
+    }
+    /** @internal */
+    _isDisposeActivated() {
+        if (this._disposeActivated)
+            return true;
+        return [...this._parentDisposeGuardsLocks.keys()].some((parent) => parent._isDisposeActivated());
+    }
+    /** @internal */
+    _activateLocksIfNeeded() {
+        if (this._preventionHandles > 0)
+            return;
+        while (this._awaitingDisposeLockCallbacks.length > 0) {
+            this._disposeActivated = true;
+            this._awaitingDisposeLockCallbacks.shift()();
+        }
+    }
+    /** @internal */
+    _updateParentDisposeGuardLocks(onlyAllowRemoval = false) {
+        if (this._preventionHandles === 0) {
+            for (const parent of this._parentDisposeGuardsLocks.keys()) {
+                const parentLock = this._parentDisposeGuardsLocks.get(parent);
+                if (parentLock == null)
+                    continue;
+                parentLock.dispose();
+                this._parentDisposeGuardsLocks.set(parent, null);
+            }
+        }
+        else if (!onlyAllowRemoval) {
+            for (const parent of this._parentDisposeGuardsLocks.keys()) {
+                if (this._parentDisposeGuardsLocks.get(parent) != null)
+                    continue;
+                this._parentDisposeGuardsLocks.set(parent, parent.createPreventDisposalHandle(true));
+            }
+        }
+    }
+    /** @internal */
+    _hasAwaitingDisposeLocks() {
+        if (this._awaitingDisposeLockCallbacks.length > 0)
+            return true;
+        return [...this._parentDisposeGuardsLocks.keys()].some((parent) => parent._hasAwaitingDisposeLocks());
+    }
+}
+export class DisposalPreventionHandle {
+    /** @internal */
+    _dispose;
+    constructor(dispose) {
+        this._dispose = dispose;
+        this.dispose = this.dispose.bind(this);
+        this[Symbol.dispose] = this[Symbol.dispose].bind(this);
+    }
+    dispose() {
+        if (this._dispose != null) {
+            this._dispose();
+            this._dispose = null;
+        }
+    }
+    [Symbol.dispose]() {
+        this.dispose();
+    }
+    get disposed() {
+        return this._dispose == null;
+    }
+    /** @internal */
+    static _create(dispose) {
+        return new DisposalPreventionHandle(dispose);
+    }
+}
+//# sourceMappingURL=DisposeGuard.js.map

package/dist/utils/DisposeGuard.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DisposeGuard.js","sourceRoot":"","sources":["../../src/utils/DisposeGuard.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAE9C,MAAM,OAAO,YAAY;IACrB,gBAAgB,CAAS,kBAAkB,GAAW,CAAC,CAAC;IACxD,gBAAgB,CAAS,6BAA6B,GAAmB,EAAE,CAAC;IAC5E,gBAAgB,CAAS,iBAAiB,GAAY,KAAK,CAAC;IAC5D,gBAAgB,CAAS,yBAAyB,GAAuD,IAAI,GAAG,EAAE,CAAC;IAEnH,YAAmB,sBAAsC,EAAE;QACvD,KAAK,MAAM,MAAM,IAAI,mBAAmB;YACpC,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAEM,qBAAqB,CAAC,MAAoB;QAC7C,IAAI,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,CAAC;YAC1C,OAAO;QAEX,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAEjD,IAAI,IAAI,CAAC,kBAAkB,GAAG,CAAC;YAC3B,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7F,CAAC;IAEM,wBAAwB,CAAC,MAAoB;QAChD,MAAM,UAAU,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAE9D,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACrB,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClD,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,kBAAkB;QAC3B,OAAO,IAAI,OAAO,CAAO,CAAC,MAAM,EAAE,EAAE;YAChC,IAAI,IAAI,CAAC,kBAAkB,GAAG,CAAC;gBAC3B,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBAC/C,CAAC;gBACF,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;gBAC9B,MAAM,EAAE,CAAC;YACb,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,2BAA2B,CAAC,wBAAiC,KAAK;QACrE,IAAI,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC,qBAAqB,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACzF,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI,CAAC;YACD,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,kBAAkB,KAAK,CAAC;gBAC7B,IAAI,CAAC,8BAA8B,EAAE,CAAC;YAE1C,MAAM,GAAG,CAAC;QACd,CAAC;QAED,OAAO,wBAAwB,CAAC,OAAO,CAAC,GAAG,EAAE;YACzC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE1B,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,gBAAgB;IACR,mBAAmB;QACvB,IAAI,IAAI,CAAC,iBAAiB;YACtB,OAAO,IAAI,CAAC;QAEhB,OAAO,CAAC,GAAG,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACrG,CAAC;IAED,gBAAgB;IACR,sBAAsB;QAC1B,IAAI,IAAI,CAAC,kBAAkB,GAAG,CAAC;YAC3B,OAAO;QAEX,OAAO,IAAI,CAAC,6BAA6B,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAC9B,IAAI,CAAC,6BAA6B,CAAC,KAAK,EAAG,EAAE,CAAC;QAClD,CAAC;IACL,CAAC;IAED,gBAAgB;IACR,8BAA8B,CAAC,mBAA4B,KAAK;QACpE,IAAI,IAAI,CAAC,kBAAkB,KAAK,CAAC,EAAE,CAAC;YAChC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,EAAE,CAAC;gBACzD,MAAM,UAAU,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAE9D,IAAI,UAAU,IAAI,IAAI;oBAClB,SAAS;gBAEb,UAAU,CAAC,OAAO,EAAE,CAAC;gBACrB,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACrD,CAAC;QACL,CAAC;aAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,EAAE,CAAC;gBACzD,IAAI,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI;oBAClD,SAAS;gBAEb,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC,CAAC;YACzF,CAAC;QACL,CAAC;IACL,CAAC;IAED,gBAAgB;IACR,wBAAwB;QAC5B,IAAI,IAAI,CAAC,6BAA6B,CAAC,MAAM,GAAG,CAAC;YAC7C,OAAO,IAAI,CAAC;QAEhB,OAAO,CAAC,GAAG,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC,CAAC;IAC1G,CAAC;CACJ;AAED,MAAM,OAAO,wBAAwB;IACjC,gBAAgB;IACR,QAAQ,CAAsB;IAEtC,YAAoB,OAAmB;QACnC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QAExB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAEM,OAAO;QACV,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACzB,CAAC;IACL,CAAC;IAEM,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;IACjC,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,OAAmB;QACrC,OAAO,IAAI,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;CACJ"}

package/dist/utils/InsufficientMemoryError.d.ts

@@ -0,0 +1,3 @@
+export declare class InsufficientMemoryError extends Error {
+    constructor(message?: string);
+}

package/dist/utils/InsufficientMemoryError.js

@@ -0,0 +1,6 @@
+export class InsufficientMemoryError extends Error {
+    constructor(message = "Insufficient memory") {
+        super(message);
+    }
+}
+//# sourceMappingURL=InsufficientMemoryError.js.map

package/dist/utils/InsufficientMemoryError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InsufficientMemoryError.js","sourceRoot":"","sources":["../../src/utils/InsufficientMemoryError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAC9C,YAAmB,UAAkB,qBAAqB;QACtD,KAAK,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;CACJ"}

package/dist/utils/LlamaText.d.ts

@@ -1,42 +1,89 @@
-import { Token, Tokenizer } from "../types.js";
-export type LlamaTextClass = {
-    <const V extends LlamaTextValue = LlamaTextValue, const V2 extends LlamaTextValue = LlamaTextValue>(strings: TemplateStringsArray | V | LlamaText<V> | readonly (LlamaText<V> | V)[], ...values: readonly (V | LlamaText<V> | V2 | LlamaText<V2> | number | boolean | readonly (LlamaText<V> | V | LlamaText<V2> | V2)[])[]): LlamaText<V | V2>;
-    fromJSON(json: LlamaTextJSON): LlamaText;
-};
-export type LlamaText<T extends LlamaTextValue = LlamaTextValue> = {
-    <const V extends LlamaTextValue = LlamaTextValue, const V2 extends LlamaTextValue = LlamaTextValue>(strings: TemplateStringsArray | V | LlamaText | readonly (LlamaText | V)[], ...values: readonly (V | V2 | LlamaText | number | boolean | readonly (LlamaText | V | V2)[])[]): LlamaText<T | V>;
-    readonly type: "LlamaText";
-    readonly values: readonly T[];
-    mapValues<V extends LlamaTextValue = LlamaTextValue>(mapper: (value: T) => V): LlamaText<V>;
-    joinValues<V extends LlamaTextValue = LlamaTextValue>(separator: LlamaText<V> | V): LlamaText<T | V>;
-    toString(): string;
-    toJSON(): LlamaTextJSON;
-    tokenize(tokenizer: Tokenizer): Token[];
+import type { Token, Tokenizer } from "../types.js";
+export type LlamaTextValue = string | SpecialTokensText | SpecialToken;
+export type LlamaTextInputValue = LlamaTextValue | LlamaText | number | boolean | readonly LlamaTextInputValue[];
+export type LlamaTextJSON = string | LlamaTextJSONValue[];
+export type LlamaTextJSONValue = string | LlamaTextSpecialTokensTextJSON | LlamaTextSpecialTokenJSON;
+export type LlamaTextSpecialTokensTextJSON = {
+    type: "specialTokensText";
+    value: string;
 };
-export type LlamaTextValue = string | SpecialToken;
-export type LlamaTextJSON = Array<LlamaTextJSONValue>;
-export type LlamaTextJSONValue = string | LlamaTextSpecialTokenJSON;
 export type LlamaTextSpecialTokenJSON = {
     type: "specialToken";
     value: string;
-    builtin?: true;
 };
-export declare const LlamaText: LlamaTextClass;
-export declare class SpecialToken {
+declare class LlamaText {
+    readonly values: readonly LlamaTextValue[];
+    /**
+     * Can also be called without `new`
+     */
+    constructor(...values: readonly LlamaTextInputValue[]);
+    concat(value: LlamaTextInputValue): LlamaText;
+    mapValues(mapper: (this: readonly LlamaTextValue[], value: LlamaTextValue, index: number, values: readonly LlamaTextValue[]) => LlamaTextInputValue): LlamaText;
+    /**
+     * Joins the values with the given separator.
+     *
+     * Note that the values are squashed when they are loaded into the `LlamaText`, so the separator is not added between adjacent strings.
+     *
+     * To add the separator on values before squashing them, use `LlamaText.joinValues` instead.
+     */
+    joinValues(separator: LlamaText | LlamaTextValue): LlamaText;
+    toString(): string;
+    toJSON(): LlamaTextJSON;
+    tokenize(tokenizer: Tokenizer, options?: "trimLeadingSpace"): Token[];
+    compare(other: LlamaText): boolean;
+    trimStart(): LlamaText;
+    trimEnd(): LlamaText;
+    includes(value: LlamaText): boolean;
+    static fromJSON(json: LlamaTextJSON): LlamaText;
+    static compare(a: LlamaText, b: LlamaText): boolean;
+    /**
+     * Attempt to convert tokens to a `LlamaText` while preserving special tokens.
+     *
+     * Non-standard special tokens that don't have a text representation are ignored.
+     */
+    static fromTokens(tokenizer: Tokenizer, tokens: Token[]): LlamaText;
+    /**
+     * Join values with the given separator before squashing adjacent strings inside the values
+     */
+    static joinValues(separator: LlamaText | string, values: readonly LlamaTextInputValue[]): LlamaText;
+    static isLlamaText(value: unknown): value is LlamaText;
+}
+type LlamaTextConstructor = Omit<typeof LlamaText, "prototype"> & {
+    new (...values: readonly LlamaTextInputValue[]): LlamaText;
+    (...values: readonly LlamaTextInputValue[]): LlamaText;
+    readonly prototype: typeof LlamaText.prototype;
+};
+declare const LlamaTextConstructor: LlamaTextConstructor;
+declare const _LlamaText: LlamaTextConstructor;
+type _LlamaText = LlamaText;
+export { _LlamaText as LlamaText, LlamaText as _LlamaText };
+export declare class SpecialTokensText {
     readonly value: string;
     constructor(value: string);
     toString(): string;
-    tokenize(tokenizer: Tokenizer): Token[];
-    toJSON(): LlamaTextSpecialTokenJSON;
-    static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;
-    static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;
+    tokenize(tokenizer: Tokenizer, trimLeadingSpace?: boolean): Token[];
+    tokenizeSpecialTokensOnly(tokenizer: Tokenizer): (string | Token)[];
+    toJSON(): LlamaTextSpecialTokensTextJSON;
+    static fromJSON(json: LlamaTextSpecialTokensTextJSON): SpecialTokensText;
+    static isSpecialTokensTextJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokensTextJSON;
+    /**
+     * Wraps the value with a `SpecialTokensText` only if `shouldWrap` is true
+     */
+    static wrapIf(shouldWrap: boolean, value: string): SpecialTokensText | string;
 }
-export type BuiltinSpecialTokenValue = "BOS" | "EOS" | "NL";
-export declare class BuiltinSpecialToken extends SpecialToken {
+export type BuiltinSpecialTokenValue = "BOS" | "EOS" | "NL" | "EOT";
+export declare class SpecialToken {
     readonly value: BuiltinSpecialTokenValue;
     constructor(value: BuiltinSpecialTokenValue);
+    toString(): BuiltinSpecialTokenValue;
     tokenize(tokenizer: Tokenizer): Token[];
     toJSON(): LlamaTextSpecialTokenJSON;
+    static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;
+    static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;
+    static getTokenToValueMap(tokenizer: Tokenizer): ReadonlyMap<Token | undefined, BuiltinSpecialTokenValue>;
 }
 export declare function isLlamaText(value: unknown): value is LlamaText;
+/**
+ * Tokenize the given input using the given tokenizer, whether it's a `string` or a `LlamaText`
+ */
 export declare function tokenizeText(text: string | LlamaText, tokenizer: Tokenizer): Token[];