node-llama-cpp 2.8.6 → 3.0.0-beta.10

package/dist/evaluator/LlamaContext/types.d.ts

@@ -0,0 +1,82 @@
+import { Token } from "../../types.js";
+import { LlamaModel } from "../LlamaModel.js";
+import { LlamaContextSequence } from "./LlamaContext.js";
+export type LlamaContextOptions = {
+    model: LlamaModel;
+    /**
+     * number of sequences for the context.
+     * Each sequence is a different "text generation process" that can run in parallel to other sequences in the same context.
+     * Although a single context has multiple sequences, the sequences are separate from each other and do not share data with each other.
+     * This is beneficial for performance, as multiple sequences can be evaluated in parallel (on the same batch).
+     */
+    sequences?: number;
+    /** If null, a random seed will be used */
+    seed?: number | null;
+    /** text context size */
+    contextSize?: number;
+    /** prompt processing batch size */
+    batchSize?: number;
+    /**
+     * number of threads to use to evaluate tokens.
+     * set to 0 to use the maximum threads supported by the current machine hardware
+     */
+    threads?: number;
+    /** control the parallel sequences processing behavior */
+    batching?: BatchingOptions;
+};
+export type LlamaContextSequenceRepeatPenalty = {
+    /** Tokens to lower the predication probability of to be the next predicted token */
+    punishTokens: Token[] | (() => Token[]);
+    /**
+     * The relative amount to lower the probability of the tokens in `punishTokens` by
+     * Defaults to `1.1`.
+     * Set to `1` to disable.
+     */
+    penalty?: number;
+    /**
+     * For n time a token is in the `punishTokens` array, lower its probability by `n * frequencyPenalty`
+     * Disabled by default (`0`).
+     * Set to a value between `0` and `1` to enable.
+     */
+    frequencyPenalty?: number;
+    /**
+     * Lower the probability of all the tokens in the `punishTokens` array by `presencePenalty`
+     * Disabled by default (`0`).
+     * Set to a value between `0` and `1` to enable.
+     */
+    presencePenalty?: number;
+};
+export type BatchingOptions = {
+    dispatchSchedule?: "nextTick" | CustomBatchingDispatchSchedule;
+    itemsPrioritizingStrategy?: "maximumParallelism" | "firstInFirstOut" | CustomBatchingPrioritizeStrategy;
+};
+export type CustomBatchingDispatchSchedule = (dispatch: () => void) => void;
+export type CustomBatchingPrioritizeStrategy = (options: {
+    items: readonly BatchItem[];
+    size: number;
+}) => PrioritizedBatchItem[];
+export type ContextShiftOptions = {
+    size?: number | ((sequence: LlamaContextSequence) => number | Promise<number>);
+    strategy?: "eraseBeginning" | ((options: {
+        sequence: LlamaContextSequence;
+        size: number;
+    }) => ContextTokensDeleteRange[] | Promise<ContextTokensDeleteRange[]>);
+};
+export type ContextTokensDeleteRange = {
+    start: number;
+    end: number;
+};
+/**
+ * 1 - low
+ *
+ * 5 - high
+ */
+export type EvaluationPriority = 1 | 2 | 3 | 4 | 5;
+export type BatchItem = {
+    readonly tokens: readonly Token[];
+    readonly evaluationPriority: EvaluationPriority;
+};
+export type PrioritizedBatchItem = {
+    item: BatchItem;
+    processAmount: number;
+};

package/dist/evaluator/LlamaContext/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/evaluator/LlamaContext/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/evaluator/LlamaContext/types.ts"],"names":[],"mappings":""}

package/dist/evaluator/LlamaContext/utils/batchItemsPrioritizingStrategies/firstInFirstOutStrategy.d.ts

@@ -0,0 +1,5 @@
+import { BatchItem, PrioritizedBatchItem } from "../../types.js";
+export declare function firstInFirstOutStrategy({ items, size }: {
+    items: readonly BatchItem[];
+    size: number;
+}): PrioritizedBatchItem[];

package/dist/evaluator/LlamaContext/utils/batchItemsPrioritizingStrategies/firstInFirstOutStrategy.js

@@ -0,0 +1,16 @@
+export function firstInFirstOutStrategy({ items, size }) {
+    const res = [];
+    const sortedItems = items
+        .slice()
+        .sort((a, b) => b.evaluationPriority - a.evaluationPriority);
+    let leftFreeTokens = size;
+    for (const item of sortedItems) {
+        const processAmount = Math.min(item.tokens.length, leftFreeTokens);
+        res.push({ item, processAmount });
+        leftFreeTokens -= processAmount;
+        if (leftFreeTokens === 0)
+            break;
+    }
+    return res;
+}
+//# sourceMappingURL=firstInFirstOutStrategy.js.map

package/dist/evaluator/LlamaContext/utils/batchItemsPrioritizingStrategies/firstInFirstOutStrategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"firstInFirstOutStrategy.js","sourceRoot":"","sources":["../../../../../src/evaluator/LlamaContext/utils/batchItemsPrioritizingStrategies/firstInFirstOutStrategy.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,uBAAuB,CAAC,EAAC,KAAK,EAAE,IAAI,EAAgD;IAChG,MAAM,GAAG,GAA2B,EAAE,CAAC;IAEvC,MAAM,WAAW,GAAG,KAAK;SACpB,KAAK,EAAE;SACP,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAEjE,IAAI,cAAc,GAAG,IAAI,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE;QAC5B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QACnE,GAAG,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,aAAa,EAAC,CAAC,CAAC;QAChC,cAAc,IAAI,aAAa,CAAC;QAEhC,IAAI,cAAc,KAAK,CAAC;YACpB,MAAM;KACb;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}

package/dist/evaluator/LlamaContext/utils/batchItemsPrioritizingStrategies/maximumParallelismStrategy.d.ts

@@ -0,0 +1,5 @@
+import { BatchItem, PrioritizedBatchItem } from "../../types.js";
+export declare function maximumParallelismStrategy({ items, size }: {
+    items: readonly BatchItem[];
+    size: number;
+}): PrioritizedBatchItem[];

package/dist/evaluator/LlamaContext/utils/batchItemsPrioritizingStrategies/maximumParallelismStrategy.js

@@ -0,0 +1,42 @@
+export function maximumParallelismStrategy({ items, size }) {
+    let leftFreeTokens = size;
+    const minTokensForEachItem = Math.floor(leftFreeTokens / items.length);
+    const res = [];
+    const clippedItems = [];
+    for (const item of items) {
+        const processAmount = Math.min(item.tokens.length, leftFreeTokens, minTokensForEachItem);
+        const prioritizeItem = { item, processAmount };
+        res.push(prioritizeItem);
+        leftFreeTokens -= processAmount;
+        if (processAmount < item.tokens.length)
+            clippedItems.push(prioritizeItem);
+        if (leftFreeTokens === 0)
+            break;
+    }
+    for (let passesLeft = 3; leftFreeTokens > 0 && clippedItems.length > 0 && passesLeft > 0; passesLeft--) {
+        const minIncreaseAmount = Math.ceil(leftFreeTokens / clippedItems.length);
+        for (let i = 0; i < clippedItems.length && leftFreeTokens > 0; i++) {
+            const prioritizeItem = clippedItems[i];
+            const unprocessedAmount = prioritizeItem.item.tokens.length - prioritizeItem.processAmount;
+            const increaseAmount = Math.min(unprocessedAmount, leftFreeTokens, minIncreaseAmount);
+            prioritizeItem.processAmount += increaseAmount;
+            if (increaseAmount === unprocessedAmount) {
+                clippedItems.splice(i, 1);
+                i--;
+            }
+        }
+    }
+    clippedItems.sort((a, b) => b.item.evaluationPriority - a.item.evaluationPriority);
+    for (let i = 0; i < clippedItems.length && leftFreeTokens > 0; i++) {
+        const prioritizeItem = clippedItems[i];
+        const unprocessedAmount = prioritizeItem.item.tokens.length - prioritizeItem.processAmount;
+        const increaseAmount = Math.min(unprocessedAmount, leftFreeTokens);
+        prioritizeItem.processAmount += increaseAmount;
+        if (increaseAmount === unprocessedAmount) {
+            clippedItems.splice(i, 1);
+            i--;
+        }
+    }
+    return res;
+}
+//# sourceMappingURL=maximumParallelismStrategy.js.map

package/dist/evaluator/LlamaContext/utils/batchItemsPrioritizingStrategies/maximumParallelismStrategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"maximumParallelismStrategy.js","sourceRoot":"","sources":["../../../../../src/evaluator/LlamaContext/utils/batchItemsPrioritizingStrategies/maximumParallelismStrategy.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,0BAA0B,CAAC,EAAC,KAAK,EAAE,IAAI,EAAgD;IACnG,IAAI,cAAc,GAAG,IAAI,CAAC;IAC1B,MAAM,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAEvE,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,MAAM,YAAY,GAA2B,EAAE,CAAC;IAEhD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QACtB,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,oBAAoB,CAAC,CAAC;QACzF,MAAM,cAAc,GAAG,EAAC,IAAI,EAAE,aAAa,EAAC,CAAC;QAE7C,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACzB,cAAc,IAAI,aAAa,CAAC;QAEhC,IAAI,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;YAClC,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAEtC,IAAI,cAAc,KAAK,CAAC;YACpB,MAAM;KACb;IAED,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,cAAc,GAAG,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,EAAE,EAAE;QACpG,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QAE1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;YAChE,MAAM,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,iBAAiB,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,aAAa,CAAC;YAC3F,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,cAAc,EAAE,iBAAiB,CAAC,CAAC;YACtF,cAAc,CAAC,aAAa,IAAI,cAAc,CAAC;YAE/C,IAAI,cAAc,KAAK,iBAAiB,EAAE;gBACtC,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC1B,CAAC,EAAE,CAAC;aACP;SACJ;KACJ;IAED,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAEnF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;QAChE,MAAM,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,iBAAiB,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,aAAa,CAAC;QAC3F,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;QACnE,cAAc,CAAC,aAAa,IAAI,cAAc,CAAC;QAE/C,IAAI,cAAc,KAAK,iBAAiB,EAAE;YACtC,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC1B,CAAC,EAAE,CAAC;SACP;KACJ;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}

package/dist/evaluator/LlamaContext/utils/resolveBatchItemsPrioritizingStrategy.d.ts

@@ -0,0 +1,2 @@
+import { BatchingOptions } from "../types.js";
+export declare function resolveBatchItemsPrioritizingStrategy(strategy: Required<BatchingOptions>["itemsPrioritizingStrategy"]): import("../types.js").CustomBatchingPrioritizeStrategy;

package/dist/evaluator/LlamaContext/utils/resolveBatchItemsPrioritizingStrategy.js

@@ -0,0 +1,13 @@
+import { maximumParallelismStrategy } from "./batchItemsPrioritizingStrategies/maximumParallelismStrategy.js";
+import { firstInFirstOutStrategy } from "./batchItemsPrioritizingStrategies/firstInFirstOutStrategy.js";
+export function resolveBatchItemsPrioritizingStrategy(strategy) {
+    if (strategy instanceof Function)
+        return strategy;
+    else if (strategy === "maximumParallelism")
+        return maximumParallelismStrategy;
+    else if (strategy === "firstInFirstOut")
+        return firstInFirstOutStrategy;
+    void (strategy);
+    throw new Error(`Unknown batch items prioritize strategy: ${strategy}`);
+}
+//# sourceMappingURL=resolveBatchItemsPrioritizingStrategy.js.map

package/dist/evaluator/LlamaContext/utils/resolveBatchItemsPrioritizingStrategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveBatchItemsPrioritizingStrategy.js","sourceRoot":"","sources":["../../../../src/evaluator/LlamaContext/utils/resolveBatchItemsPrioritizingStrategy.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,0BAA0B,EAAC,MAAM,kEAAkE,CAAC;AAC5G,OAAO,EAAC,uBAAuB,EAAC,MAAM,+DAA+D,CAAC;AAEtG,MAAM,UAAU,qCAAqC,CAAC,QAAgE;IAClH,IAAI,QAAQ,YAAY,QAAQ;QAC5B,OAAO,QAAQ,CAAC;SACf,IAAI,QAAQ,KAAK,oBAAoB;QACtC,OAAO,0BAA0B,CAAC;SACjC,IAAI,QAAQ,KAAK,iBAAiB;QACnC,OAAO,uBAAuB,CAAC;IAEnC,KAAK,CAAC,QAAwB,CAAC,CAAC;IAEhC,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;AAC5E,CAAC"}

package/dist/evaluator/LlamaEmbeddingContext.d.ts

@@ -0,0 +1,37 @@
+import { EventRelay } from "lifecycle-utils";
+import { Token } from "../types.js";
+import { LlamaText } from "../utils/LlamaText.js";
+import { LlamaModel } from "./LlamaModel.js";
+export type LlamaEmbeddingContextOptions = {
+    model: LlamaModel;
+    /** text context size */
+    contextSize?: number;
+    /** prompt processing batch size */
+    batchSize?: number;
+    /**
+     * number of threads to use to evaluate tokens.
+     * set to 0 to use the maximum threads supported by the current machine hardware
+     */
+    threads?: number;
+};
+export declare class LlamaEmbeddingContext {
+    readonly onDispose: EventRelay<void>;
+    constructor({ model, contextSize, batchSize, threads }: LlamaEmbeddingContextOptions);
+    getEmbeddingFor(input: Token[] | string | LlamaText): Promise<LlamaEmbedding>;
+    dispose(): void;
+    /** @hidden */
+    [Symbol.dispose](): void;
+    get disposed(): boolean;
+}
+export type LlamaEmbeddingJSON = {
+    type: "LlamaEmbedding";
+    vector: number[];
+};
+export declare class LlamaEmbedding {
+    readonly vector: number[];
+    constructor({ vector }: {
+        vector: number[];
+    });
+    toJSON(): LlamaEmbeddingJSON;
+    static fromJSON(json: LlamaEmbeddingJSON): LlamaEmbedding;
+}

package/dist/evaluator/LlamaEmbeddingContext.js

@@ -0,0 +1,78 @@
+import { DisposeAggregator, EventRelay, withLock } from "lifecycle-utils";
+import { tokenizeInput } from "../utils/tokenizeInput.js";
+import { LlamaContext } from "./LlamaContext/LlamaContext.js";
+export class LlamaEmbeddingContext {
+    /** @internal */ _llamaContext;
+    /** @internal */ _sequence;
+    /** @internal */ _disposeAggregator = new DisposeAggregator();
+    onDispose = new EventRelay();
+    constructor({ model, contextSize = model.trainContextSize, batchSize = contextSize, threads = 6 }) {
+        const resolvedContextSize = Math.min(contextSize, model.trainContextSize);
+        const resolvedBatchSize = Math.min(batchSize, resolvedContextSize);
+        this._llamaContext = new LlamaContext({
+            model,
+            contextSize: resolvedContextSize,
+            batchSize: resolvedBatchSize,
+            threads,
+            _embedding: true,
+            _noSeed: true
+        });
+        this._sequence = this._llamaContext.getSequence();
+        this._disposeAggregator.add(this._llamaContext.onDispose.createListener(() => {
+            this._disposeAggregator.dispose();
+        }));
+        this._disposeAggregator.add(this.onDispose.dispatchEvent);
+        this._disposeAggregator.add(() => {
+            this._llamaContext.dispose();
+        });
+    }
+    async getEmbeddingFor(input) {
+        const resolvedInput = tokenizeInput(input, this._llamaContext.model.tokenize);
+        if (resolvedInput.length > this._llamaContext.contextSize)
+            throw new Error("Input is longer than the context size. " +
+                "Try to increase the context size or use another model that supports longer contexts.");
+        else if (resolvedInput.length === 0)
+            return new LlamaEmbedding({ vector: [] });
+        return await withLock(this, "evaluate", async () => {
+            await this._sequence.eraseContextTokenRanges([{
+                    start: 0,
+                    end: this._sequence.nextTokenIndex
+                }]);
+            await this._sequence.evaluateWithoutGeneratingNewTokens(resolvedInput);
+            const embedding = this._llamaContext._ctx.getEmbedding();
+            const embeddingVector = Array.from(embedding);
+            return new LlamaEmbedding({ vector: embeddingVector });
+        });
+    }
+    dispose() {
+        this._disposeAggregator.dispose();
+    }
+    /** @hidden */
+    [Symbol.dispose]() {
+        return this.dispose();
+    }
+    get disposed() {
+        return this._llamaContext.disposed;
+    }
+}
+export class LlamaEmbedding {
+    vector;
+    constructor({ vector }) {
+        this.vector = vector;
+    }
+    toJSON() {
+        return {
+            type: "LlamaEmbedding",
+            vector: this.vector
+        };
+    }
+    static fromJSON(json) {
+        if (json == null || json.type !== "LlamaEmbedding" || !(json.vector instanceof Array) ||
+            json.vector.some(v => typeof v !== "number"))
+            throw new Error("Invalid LlamaEmbedding JSON");
+        return new LlamaEmbedding({
+            vector: json.vector
+        });
+    }
+}
+//# sourceMappingURL=LlamaEmbeddingContext.js.map

package/dist/evaluator/LlamaEmbeddingContext.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaEmbeddingContext.js","sourceRoot":"","sources":["../../src/evaluator/LlamaEmbeddingContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAE,UAAU,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AAGxE,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AAExD,OAAO,EAAC,YAAY,EAAuB,MAAM,gCAAgC,CAAC;AAkBlF,MAAM,OAAO,qBAAqB;IAC9B,gBAAgB,CAAkB,aAAa,CAAe;IAC9D,gBAAgB,CAAkB,SAAS,CAAuB;IAClE,gBAAgB,CAAkB,kBAAkB,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE/D,SAAS,GAAG,IAAI,UAAU,EAAQ,CAAC;IAEnD,YAAmB,EACf,KAAK,EACL,WAAW,GAAG,KAAK,CAAC,gBAAgB,EACpC,SAAS,GAAG,WAAW,EACvB,OAAO,GAAG,CAAC,EACgB;QAC3B,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAC1E,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;QAEnE,IAAI,CAAC,aAAa,GAAG,IAAI,YAAY,CAAC;YAClC,KAAK;YACL,WAAW,EAAE,mBAAmB;YAChC,SAAS,EAAE,iBAAiB;YAC5B,OAAO;YACP,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAElD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CACvB,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,EAAE;YAC7C,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;QACtC,CAAC,CAAC,CACL,CAAC;QACF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAC1D,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE;YAC7B,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;QACjC,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,KAAmC;QAC5D,MAAM,aAAa,GAAG,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAE9E,IAAI,aAAa,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW;YACrD,MAAM,IAAI,KAAK,CACX,yCAAyC;gBACzC,sFAAsF,CACzF,CAAC;aACD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAC/B,OAAO,IAAI,cAAc,CAAC,EAAC,MAAM,EAAE,EAAE,EAAC,CAAC,CAAC;QAE5C,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;oBAC1C,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;iBACrC,CAAC,CAAC,CAAC;YAEJ,MAAM,IAAI,CAAC,SAAS,CAAC,kCAAkC,CAAC,aAAa,CAAC,CAAC;YAEvE,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACzD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAE9C,OAAO,IAAI,cAAc,CAAC,EAAC,MAAM,EAAE,eAAe,EAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,OAAO;QACV,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;IACtC,CAAC;IAED,cAAc;IACP,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC;IACvC,CAAC;CACJ;AAOD,MAAM,OAAO,cAAc;IACP,MAAM,CAAW;IAEjC,YAAmB,EAAC,MAAM,EAAqB;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAEM,MAAM;QACT,OAAO;YACH,IAAI,EAAE,gBAAgB;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,IAAwB;QAC3C,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,YAAY,KAAK,CAAC;YACjF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;YAE5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAEnD,OAAO,IAAI,cAAc,CAAC;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC,CAAC;IACP,CAAC;CACJ"}

package/dist/evaluator/LlamaGrammar.d.ts

@@ -0,0 +1,30 @@
+import { LlamaText } from "../utils/LlamaText.js";
+import { StopGenerationTrigger } from "../utils/StopGenerationDetector.js";
+import { Llama } from "../bindings/Llama.js";
+export type LlamaGrammarOptions = {
+    llama: Llama;
+    /** GBNF grammar */
+    grammar: string;
+    /** print the grammar to stdout */
+    printGrammar?: boolean;
+    /** Consider any of these as EOS for the generated text. Only supported by `LlamaChat` and `LlamaChatSession` */
+    stopGenerationTriggers?: readonly (StopGenerationTrigger | LlamaText)[];
+    /** Trim whitespace from the end of the generated text. Only supported by `LlamaChat` and `LlamaChatSession` */
+    trimWhitespaceSuffix?: boolean;
+};
+export declare class LlamaGrammar {
+    private readonly _stopGenerationTriggers;
+    private readonly _trimWhitespaceSuffix;
+    private readonly _grammarText;
+    /**
+     * > GBNF files are supported.
+     * > More info here: [github:ggerganov/llama.cpp:grammars/README.md](
+     * > https://github.com/ggerganov/llama.cpp/blob/f5fe98d11bdf9e7797bcfb05c0c3601ffc4b9d26/grammars/README.md)
+     * @param options
+     */
+    constructor({ llama, grammar, stopGenerationTriggers, trimWhitespaceSuffix, printGrammar }: LlamaGrammarOptions);
+    get grammar(): string;
+    get stopGenerationTriggers(): readonly (StopGenerationTrigger | LlamaText)[];
+    get trimWhitespaceSuffix(): boolean;
+    static getFor(llama: Llama, type: "json" | "list" | "arithmetic" | "japanese" | "chess"): Promise<LlamaGrammar>;
+}

package/dist/{llamaEvaluator → evaluator}/LlamaGrammar.js

@@ -1,50 +1,46 @@
 import path from "path";
 import fs from "fs-extra";
 import { getGrammarsFolder } from "../utils/getGrammarsFolder.js";
-import { LLAMAGrammar } from "./LlamaBins.js";
+import { LlamaText } from "../utils/LlamaText.js";
 export class LlamaGrammar {
-    /** @internal */
-    _grammar;
-    _stopStrings;
+    /** @internal */ _llama;
+    /** @internal */ _grammar;
+    _stopGenerationTriggers;
     _trimWhitespaceSuffix;
     _grammarText;
     /**
      * > GBNF files are supported.
      * > More info here: [github:ggerganov/llama.cpp:grammars/README.md](
      * > https://github.com/ggerganov/llama.cpp/blob/f5fe98d11bdf9e7797bcfb05c0c3601ffc4b9d26/grammars/README.md)
-     * @param {object} options
-     * @param {string} options.grammar - GBNF grammar
-     * @param {string[]} [options.stopStrings] - Consider any of these texts as EOS for the generated out.
-     * Only supported by `LlamaChatSession`
-     * @param {boolean} [options.trimWhitespaceSuffix] - Trim whitespace from the end of the generated text.
-     * Only supported by `LlamaChatSession`
-     * @param {boolean} [options.printGrammar] - print the grammar to stdout
+     * @param options
      */
-    constructor({ grammar, stopStrings = [], trimWhitespaceSuffix = false, printGrammar = false }) {
-        this._grammar = new LLAMAGrammar(grammar, {
+    constructor({ llama, grammar, stopGenerationTriggers = [], trimWhitespaceSuffix = false, printGrammar = false }) {
+        this._llama = llama;
+        this._grammar = new this._llama._bindings.AddonGrammar(grammar, {
             printGrammar
         });
-        this._stopStrings = stopStrings ?? [];
+        this._stopGenerationTriggers = stopGenerationTriggers ?? [];
         this._trimWhitespaceSuffix = trimWhitespaceSuffix;
         this._grammarText = grammar;
     }
     get grammar() {
         return this._grammarText;
     }
-    get stopStrings() {
-        return this._stopStrings;
+    get stopGenerationTriggers() {
+        return this._stopGenerationTriggers;
     }
     get trimWhitespaceSuffix() {
         return this._trimWhitespaceSuffix;
     }
-    static async getFor(type) {
-        const grammarsFolder = await getGrammarsFolder();
+    static async getFor(llama, type) {
+        const grammarsFolder = await getGrammarsFolder(llama.buildType);
         const grammarFile = path.join(grammarsFolder, type + ".gbnf");
         if (await fs.pathExists(grammarFile)) {
             const grammar = await fs.readFile(grammarFile, "utf8");
             return new LlamaGrammar({
+                llama,
                 grammar,
-                stopStrings: ["\n".repeat(10)],
+                stopGenerationTriggers: [LlamaText(["\n".repeat(10)])],
                 trimWhitespaceSuffix: true
             });
         }

package/dist/evaluator/LlamaGrammar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaGrammar.js","sourceRoot":"","sources":["../../src/evaluator/LlamaGrammar.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAsBhD,MAAM,OAAO,YAAY;IACrB,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAiB,QAAQ,CAAe;IACvC,uBAAuB,CAAiD;IACxE,qBAAqB,CAAU;IAC/B,YAAY,CAAS;IAEtC;;;;;OAKG;IACH,YAAmB,EACf,KAAK,EAAE,OAAO,EAAE,sBAAsB,GAAG,EAAE,EAAE,oBAAoB,GAAG,KAAK,EAAE,YAAY,GAAG,KAAK,EAC7E;QAClB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE;YAC5D,YAAY;SACf,CAAC,CAAC;QACH,IAAI,CAAC,uBAAuB,GAAG,sBAAsB,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,sBAAsB;QAC7B,OAAO,IAAI,CAAC,uBAAuB,CAAC;IACxC,CAAC;IAED,IAAW,oBAAoB;QAC3B,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAY,EAAE,IAA2D;QAChG,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEhE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,OAAO,CAAC,CAAC;QAE9D,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE;YAClC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACvD,OAAO,IAAI,YAAY,CAAC;gBACpB,KAAK;gBACL,OAAO;gBACP,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACtD,oBAAoB,EAAE,IAAI;aAC7B,CAAC,CAAC;SACN;QAED,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,uBAAuB,cAAc,GAAG,CAAC,CAAC;IAC5F,CAAC;CACJ"}

package/dist/evaluator/LlamaGrammarEvaluationState.d.ts

@@ -0,0 +1,15 @@
+import { LlamaGrammar } from "./LlamaGrammar.js";
+export type LlamaGrammarEvaluationStateOptions = {
+    grammar: LlamaGrammar;
+};
+/**
+ * Grammar evaluation state is used to track the model response to determine the next allowed characters for the model to generate.
+ * Create a new grammar evaluation state for every response you generate with the model.
+ * This is only needed when using the `LlamaContext` class directly, as `LlamaChatSession` already handles this for you.
+ */
+export declare class LlamaGrammarEvaluationState {
+    /**
+     * @param options
+     */
+    constructor({ grammar }: LlamaGrammarEvaluationStateOptions);
+}

package/dist/evaluator/LlamaGrammarEvaluationState.js

@@ -0,0 +1,17 @@
+/**
+ * Grammar evaluation state is used to track the model response to determine the next allowed characters for the model to generate.
+ * Create a new grammar evaluation state for every response you generate with the model.
+ * This is only needed when using the `LlamaContext` class directly, as `LlamaChatSession` already handles this for you.
+ */
+export class LlamaGrammarEvaluationState {
+    /** @internal */ _llama;
+    /** @internal */ _state;
+    /**
+     * @param options
+     */
+    constructor({ grammar }) {
+        this._llama = grammar._llama;
+        this._state = new grammar._llama._bindings.AddonGrammarEvaluationState(grammar._grammar);
+    }
+}
+//# sourceMappingURL=LlamaGrammarEvaluationState.js.map

package/dist/evaluator/LlamaGrammarEvaluationState.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaGrammarEvaluationState.js","sourceRoot":"","sources":["../../src/evaluator/LlamaGrammarEvaluationState.ts"],"names":[],"mappings":"AASA;;;;GAIG;AACH,MAAM,OAAO,2BAA2B;IACpC,gBAAgB,CAAiB,MAAM,CAAQ;IAC/C,gBAAgB,CAAiB,MAAM,CAA8B;IAErE;;OAEG;IACH,YAAmB,EAAC,OAAO,EAAqC;QAC5D,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,2BAA2B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7F,CAAC;CACJ"}

package/dist/{llamaEvaluator → evaluator}/LlamaJsonSchemaGrammar.d.ts

@@ -1,7 +1,8 @@
 import { GbnfJsonSchema, GbnfJsonSchemaToType } from "../utils/gbnfJson/types.js";
+import { Llama } from "../bindings/Llama.js";
 import { LlamaGrammar } from "./LlamaGrammar.js";
 export declare class LlamaJsonSchemaGrammar<const T extends Readonly<GbnfJsonSchema>> extends LlamaGrammar {
     private readonly _schema;
-    constructor(schema: T);
+    constructor(llama: Llama, schema: T);
     parse(json: string): GbnfJsonSchemaToType<T>;
 }

package/dist/{llamaEvaluator → evaluator}/LlamaJsonSchemaGrammar.js

@@ -1,13 +1,15 @@
 import { getGbnfGrammarForGbnfJsonSchema } from "../utils/getGbnfGrammarForGbnfJsonSchema.js";
 import { validateObjectAgainstGbnfSchema } from "../utils/gbnfJson/utils/validateObjectAgainstGbnfSchema.js";
+import { LlamaText } from "../utils/LlamaText.js";
 import { LlamaGrammar } from "./LlamaGrammar.js";
 export class LlamaJsonSchemaGrammar extends LlamaGrammar {
     _schema;
-    constructor(schema) {
+    constructor(llama, schema) {
         const grammar = getGbnfGrammarForGbnfJsonSchema(schema);
         super({
+            llama,
             grammar,
-            stopStrings: ["\n".repeat(4)],
+            stopGenerationTriggers: [LlamaText(["\n".repeat(4)])],
             trimWhitespaceSuffix: true
         });
         this._schema = schema;

package/dist/evaluator/LlamaJsonSchemaGrammar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaJsonSchemaGrammar.js","sourceRoot":"","sources":["../../src/evaluator/LlamaJsonSchemaGrammar.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,+BAA+B,EAAC,MAAM,6CAA6C,CAAC;AAC5F,OAAO,EAAC,+BAA+B,EAAC,MAAM,4DAA4D,CAAC;AAC3G,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAE/C,MAAM,OAAO,sBAAiE,SAAQ,YAAY;IAC7E,OAAO,CAAI;IAE5B,YAAmB,KAAY,EAAE,MAAS;QACtC,MAAM,OAAO,GAAG,+BAA+B,CAAC,MAAM,CAAC,CAAC;QAExD,KAAK,CAAC;YACF,KAAK;YACL,OAAO;YACP,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,oBAAoB,EAAE,IAAI;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEpC,+BAA+B,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAE1D,OAAO,UAAU,CAAC;IACtB,CAAC;CACJ"}

package/dist/evaluator/LlamaModel.d.ts

@@ -0,0 +1,120 @@
+import { EventRelay } from "lifecycle-utils";
+import { Token } from "../types.js";
+import { ModelTypeDescription } from "../bindings/AddonTypes.js";
+import { Llama } from "../bindings/Llama.js";
+import type { BuiltinSpecialTokenValue } from "../utils/LlamaText.js";
+export type LlamaModelOptions = {
+    llama: Llama;
+    /** path to the model on the filesystem */
+    modelPath: string;
+    /** number of layers to store in VRAM */
+    gpuLayers?: number;
+    /** only load the vocabulary, no weights */
+    vocabOnly?: boolean;
+    /** use mmap if possible */
+    useMmap?: boolean;
+    /** force system to keep model in RAM */
+    useMlock?: boolean;
+};
+export declare class LlamaModel {
+    readonly onDispose: EventRelay<void>;
+    /**
+     * > options source:
+     * > [github:ggerganov/llama.cpp/llama.h](
+     * > https://github.com/ggerganov/llama.cpp/blob/05816027d649f977468fc804cdb54e99eac246d1/llama.h#L161) (`struct llama_model_params`)
+     * @param options
+     * @param options.modelPath - path to the model on the filesystem
+     * @param [options.gpuLayers] - number of layers to store in VRAM
+     * @param [options.vocabOnly] - only load the vocabulary, no weights
+     * @param [options.useMmap] - use mmap if possible
+     * @param [options.useMlock] - force system to keep model in RAM
+     */
+    constructor({ llama, modelPath, gpuLayers, vocabOnly, useMmap, useMlock }: LlamaModelOptions);
+    dispose(): void;
+    /** @hidden */
+    [Symbol.dispose](): void;
+    get disposed(): boolean;
+    get tokens(): LlamaModelTokens;
+    get filename(): string | undefined;
+    /**
+     * Transform text into tokens that can be fed to the model
+     * @param text - the text to tokenize
+     * @param [specialTokens] - if set to true, text that correspond to special tokens will be tokenized to those tokens.
+     * For example, `<s>` will be tokenized to the BOS token if `specialTokens` is set to `true`,
+     * otherwise it will be tokenized to tokens that corresponds to the plaintext `<s>` string.
+     */
+    tokenize(text: string, specialTokens?: boolean): Token[];
+    tokenize(text: BuiltinSpecialTokenValue, specialTokens: "builtin"): Token[];
+    /** Transform tokens into text */
+    detokenize(tokens: readonly Token[]): string;
+    /** @hidden `ModelTypeDescription` type alias is too long in the documentation */
+    get typeDescription(): ModelTypeDescription;
+    /** The context size the model was trained on */
+    get trainContextSize(): number;
+}
+export declare class LlamaModelTokens {
+    private constructor();
+    /**
+     * @returns infill tokens
+     */
+    get infill(): LlamaModelInfillTokens;
+    /**
+     * @returns The BOS (Beginning Of Sequence) token.
+     */
+    get bos(): Token | null;
+    /**
+     * @returns The EOS (End Of Sequence) token.
+     */
+    get eos(): Token | null;
+    /**
+     * @returns The NL (New Line) token.
+     */
+    get nl(): Token | null;
+    /**
+     * @returns The BOS (Beginning Of Sequence) token as a string.
+     */
+    get bosString(): string | null;
+    /**
+     * @returns The EOS (End Of Sequence) token as a string.
+     */
+    get eosString(): string | null;
+    /**
+     * @returns The NL (New Line) token as a string.
+     */
+    get nlString(): string | null;
+}
+export declare class LlamaModelInfillTokens {
+    private constructor();
+    /**
+     * @returns The beginning of infill prefix token.
+     */
+    get prefix(): Token | null;
+    /**
+     * @returns The beginning of infill middle token.
+     */
+    get middle(): Token | null;
+    /**
+     * @returns The beginning of infill suffix token.
+     */
+    get suffix(): Token | null;
+    /**
+     * @returns End of infill middle token (End Of Text).
+     */
+    get eot(): Token | null;
+    /**
+     * @returns The beginning of infill prefix token as a string.
+     */
+    get prefixString(): string | null;
+    /**
+     * @returns The beginning of infill middle token as a string.
+     */
+    get middleString(): string | null;
+    /**
+     * @returns The beginning of infill suffix token as a string.
+     */
+    get suffixString(): string | null;
+    /**
+     * @returns End of infill middle token (End Of Text) as a string.
+     */
+    get eotString(): string | null;
+}