@shipworthy/ai-sdk-llama-cpp 0.2.0

package/dist/llama-cpp-language-model.js

@@ -0,0 +1,221 @@
+import { loadModel, unloadModel, generate, generateStream, isModelLoaded, } from "./native-binding.js";
+export function convertFinishReason(reason) {
+    let unified;
+    switch (reason) {
+        case "stop":
+            unified = "stop";
+            break;
+        case "length":
+            unified = "length";
+            break;
+        default:
+            unified = "other";
+    }
+    return { unified, raw: reason };
+}
+export function convertUsage(promptTokens, completionTokens) {
+    return {
+        inputTokens: {
+            total: promptTokens,
+            noCache: undefined,
+            cacheRead: undefined,
+            cacheWrite: undefined,
+        },
+        outputTokens: {
+            total: completionTokens,
+            text: completionTokens,
+            reasoning: undefined,
+        },
+    };
+}
+/**
+ * Convert AI SDK messages to simple role/content format for the native layer.
+ * The native layer will apply the appropriate chat template.
+ */
+export function convertMessages(messages) {
+    const result = [];
+    for (const message of messages) {
+        switch (message.role) {
+            case "system":
+                result.push({
+                    role: "system",
+                    content: message.content,
+                });
+                break;
+            case "user":
+                // Extract text content from user messages
+                let userContent = "";
+                for (const part of message.content) {
+                    if (part.type === "text") {
+                        userContent += part.text;
+                    }
+                    // Note: File parts are not supported in this implementation
+                }
+                result.push({
+                    role: "user",
+                    content: userContent,
+                });
+                break;
+            case "assistant":
+                // Extract text content from assistant messages
+                let assistantContent = "";
+                for (const part of message.content) {
+                    if (part.type === "text") {
+                        assistantContent += part.text;
+                    }
+                }
+                result.push({
+                    role: "assistant",
+                    content: assistantContent,
+                });
+                break;
+            case "tool":
+                // Tool results are not supported in this implementation
+                break;
+        }
+    }
+    return result;
+}
+export class LlamaCppLanguageModel {
+    specificationVersion = "v3";
+    provider = "llama.cpp";
+    modelId;
+    /**
+     * Supported URL patterns - empty since we only support local files
+     */
+    supportedUrls = {};
+    modelHandle = null;
+    config;
+    initPromise = null;
+    constructor(config) {
+        this.config = config;
+        this.modelId = config.modelPath;
+    }
+    async ensureModelLoaded() {
+        if (this.modelHandle !== null && isModelLoaded(this.modelHandle)) {
+            return this.modelHandle;
+        }
+        if (this.initPromise) {
+            await this.initPromise;
+            if (this.modelHandle !== null) {
+                return this.modelHandle;
+            }
+        }
+        this.initPromise = (async () => {
+            const options = {
+                modelPath: this.config.modelPath,
+                contextSize: this.config.contextSize ?? 2048,
+                gpuLayers: this.config.gpuLayers ?? 99,
+                threads: this.config.threads ?? 4,
+                debug: this.config.debug ?? false,
+                chatTemplate: this.config.chatTemplate ?? "auto",
+            };
+            this.modelHandle = await loadModel(options);
+        })();
+        await this.initPromise;
+        this.initPromise = null;
+        if (this.modelHandle === null) {
+            throw new Error("Failed to load model");
+        }
+        return this.modelHandle;
+    }
+    async dispose() {
+        if (this.modelHandle !== null) {
+            unloadModel(this.modelHandle);
+            this.modelHandle = null;
+        }
+    }
+    async doGenerate(options) {
+        const handle = await this.ensureModelLoaded();
+        const messages = convertMessages(options.prompt);
+        const generateOptions = {
+            messages,
+            maxTokens: options.maxOutputTokens ?? 256,
+            temperature: options.temperature ?? 0.7,
+            topP: options.topP ?? 0.9,
+            topK: options.topK ?? 40,
+            stopSequences: options.stopSequences,
+        };
+        const result = await generate(handle, generateOptions);
+        // Build content array with text content
+        const content = [
+            {
+                type: "text",
+                text: result.text,
+                providerMetadata: undefined,
+            },
+        ];
+        const warnings = [];
+        return {
+            content,
+            finishReason: convertFinishReason(result.finishReason),
+            usage: convertUsage(result.promptTokens, result.completionTokens),
+            warnings,
+            request: {
+                body: generateOptions,
+            },
+        };
+    }
+    async doStream(options) {
+        const handle = await this.ensureModelLoaded();
+        const messages = convertMessages(options.prompt);
+        const generateOptions = {
+            messages,
+            maxTokens: options.maxOutputTokens ?? 256,
+            temperature: options.temperature ?? 0.7,
+            topP: options.topP ?? 0.9,
+            topK: options.topK ?? 40,
+            stopSequences: options.stopSequences,
+        };
+        const textId = crypto.randomUUID();
+        const stream = new ReadableStream({
+            start: async (controller) => {
+                try {
+                    // Emit stream start
+                    controller.enqueue({
+                        type: "stream-start",
+                        warnings: [],
+                    });
+                    // Emit text start
+                    controller.enqueue({
+                        type: "text-start",
+                        id: textId,
+                    });
+                    const result = await generateStream(handle, generateOptions, (token) => {
+                        controller.enqueue({
+                            type: "text-delta",
+                            id: textId,
+                            delta: token,
+                        });
+                    });
+                    // Emit text end
+                    controller.enqueue({
+                        type: "text-end",
+                        id: textId,
+                    });
+                    // Emit finish
+                    controller.enqueue({
+                        type: "finish",
+                        finishReason: convertFinishReason(result.finishReason),
+                        usage: convertUsage(result.promptTokens, result.completionTokens),
+                    });
+                    controller.close();
+                }
+                catch (error) {
+                    controller.enqueue({
+                        type: "error",
+                        error,
+                    });
+                    controller.close();
+                }
+            },
+        });
+        return {
+            stream,
+            request: {
+                body: generateOptions,
+            },
+        };
+    }
+}
+//# sourceMappingURL=llama-cpp-language-model.js.map

package/dist/llama-cpp-language-model.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-cpp-language-model.js","sourceRoot":"","sources":["../src/llama-cpp-language-model.ts"],"names":[],"mappings":"AAaA,OAAO,EACL,SAAS,EACT,WAAW,EACX,QAAQ,EACR,cAAc,EACd,aAAa,GAId,MAAM,qBAAqB,CAAC;AAgC7B,MAAM,UAAU,mBAAmB,CACjC,MAAc;IAEd,IAAI,OAA+C,CAAC;IACpD,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,MAAM;YACT,OAAO,GAAG,MAAM,CAAC;YACjB,MAAM;QACR,KAAK,QAAQ;YACX,OAAO,GAAG,QAAQ,CAAC;YACnB,MAAM;QACR;YACE,OAAO,GAAG,OAAO,CAAC;IACtB,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,YAAoB,EACpB,gBAAwB;IAExB,OAAO;QACL,WAAW,EAAE;YACX,KAAK,EAAE,YAAY;YACnB,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,SAAS;YACpB,UAAU,EAAE,SAAS;SACtB;QACD,YAAY,EAAE;YACZ,KAAK,EAAE,gBAAgB;YACvB,IAAI,EAAE,gBAAgB;YACtB,SAAS,EAAE,SAAS;SACrB;KACF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,QAAkC;IAChE,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,QAAQ;gBACX,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,OAAO,CAAC,OAAO;iBACzB,CAAC,CAAC;gBACH,MAAM;YACR,KAAK,MAAM;gBACT,0CAA0C;gBAC1C,IAAI,WAAW,GAAG,EAAE,CAAC;gBACrB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACnC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;wBACzB,WAAW,IAAI,IAAI,CAAC,IAAI,CAAC;oBAC3B,CAAC;oBACD,4DAA4D;gBAC9D,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,WAAW;iBACrB,CAAC,CAAC;gBACH,MAAM;YACR,KAAK,WAAW;gBACd,+CAA+C;gBAC/C,IAAI,gBAAgB,GAAG,EAAE,CAAC;gBAC1B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACnC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;wBACzB,gBAAgB,IAAI,IAAI,CAAC,IAAI,CAAC;oBAChC,CAAC;gBACH,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,gBAAgB;iBAC1B,CAAC,CAAC;gBACH,MAAM;YACR,KAAK,MAAM;gBACT,wDAAwD;gBACxD,MAAM;QACV,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,qBAAqB;IACvB,oBAAoB,GAAG,IAAa,CAAC;IACrC,QAAQ,GAAG,WAAW,CAAC;IACvB,OAAO,CAAS;IAEzB;;OAEG;IACM,aAAa,GAA6B,EAAE,CAAC;IAE9C,WAAW,GAAkB,IAAI,CAAC;IACzB,MAAM,CAAsB;IACrC,WAAW,GAAyB,IAAI,CAAC;IAEjD,YAAY,MAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,IAAI,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACjE,OAAO,IAAI,CAAC,WAAW,CAAC;QAC1B,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,WAAW,CAAC;YACvB,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;gBAC9B,OAAO,IAAI,CAAC,WAAW,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,MAAM,OAAO,GAAqB;gBAChC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBAChC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI;gBAC5C,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE;gBACtC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC;gBACjC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,KAAK;gBACjC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,MAAM;aACjD,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;QAC9C,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,IAAI,CAAC,WAAW,CAAC;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAC9B,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC9B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CACd,OAAmC;QAEnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE9C,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,eAAe,GAAoB;YACvC,QAAQ;YACR,SAAS,EAAE,OAAO,CAAC,eAAe,IAAI,GAAG;YACzC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,GAAG;YACvC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,GAAG;YACzB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE;YACxB,aAAa,EAAE,OAAO,CAAC,aAAa;SACrC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAEvD,wCAAwC;QACxC,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,gBAAgB,EAAE,SAAS;aAC5B;SACF,CAAC;QAEF,MAAM,QAAQ,GAAsB,EAAE,CAAC;QAEvC,OAAO;YACL,OAAO;YACP,YAAY,EAAE,mBAAmB,CAAC,MAAM,CAAC,YAAY,CAAC;YACtD,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,gBAAgB,CAAC;YACjE,QAAQ;YACR,OAAO,EAAE;gBACP,IAAI,EAAE,eAAe;aACtB;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,OAAmC;QAEnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE9C,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,eAAe,GAAoB;YACvC,QAAQ;YACR,SAAS,EAAE,OAAO,CAAC,eAAe,IAAI,GAAG;YACzC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,GAAG;YACvC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,GAAG;YACzB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE;YACxB,aAAa,EAAE,OAAO,CAAC,aAAa;SACrC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QAEnC,MAAM,MAAM,GAAG,IAAI,cAAc,CAA4B;YAC3D,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;gBAC1B,IAAI,CAAC;oBACH,oBAAoB;oBACpB,UAAU,CAAC,OAAO,CAAC;wBACjB,IAAI,EAAE,cAAc;wBACpB,QAAQ,EAAE,EAAE;qBACb,CAAC,CAAC;oBAEH,kBAAkB;oBAClB,UAAU,CAAC,OAAO,CAAC;wBACjB,IAAI,EAAE,YAAY;wBAClB,EAAE,EAAE,MAAM;qBACX,CAAC,CAAC;oBAEH,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC,MAAM,EACN,eAAe,EACf,CAAC,KAAK,EAAE,EAAE;wBACR,UAAU,CAAC,OAAO,CAAC;4BACjB,IAAI,EAAE,YAAY;4BAClB,EAAE,EAAE,MAAM;4BACV,KAAK,EAAE,KAAK;yBACb,CAAC,CAAC;oBACL,CAAC,CACF,CAAC;oBAEF,gBAAgB;oBAChB,UAAU,CAAC,OAAO,CAAC;wBACjB,IAAI,EAAE,UAAU;wBAChB,EAAE,EAAE,MAAM;qBACX,CAAC,CAAC;oBAEH,cAAc;oBACd,UAAU,CAAC,OAAO,CAAC;wBACjB,IAAI,EAAE,QAAQ;wBACd,YAAY,EAAE,mBAAmB,CAAC,MAAM,CAAC,YAAY,CAAC;wBACtD,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,gBAAgB,CAAC;qBAClE,CAAC,CAAC;oBAEH,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,UAAU,CAAC,OAAO,CAAC;wBACjB,IAAI,EAAE,OAAO;wBACb,KAAK;qBACN,CAAC,CAAC;oBACH,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,OAAO;YACL,MAAM;YACN,OAAO,EAAE;gBACP,IAAI,EAAE,eAAe;aACtB;SACF,CAAC;IACJ,CAAC;CACF"}

package/dist/llama-cpp-provider.d.ts

@@ -0,0 +1,82 @@
+import { LlamaCppLanguageModel } from "./llama-cpp-language-model.js";
+import { LlamaCppEmbeddingModel } from "./llama-cpp-embedding-model.js";
+export interface LlamaCppProviderConfig {
+    /**
+     * Path to the GGUF model file.
+     */
+    modelPath: string;
+    /**
+     * Maximum context size (default: 2048).
+     */
+    contextSize?: number;
+    /**
+     * Number of layers to offload to GPU (default: 99, meaning all layers).
+     * Set to 0 to disable GPU acceleration.
+     */
+    gpuLayers?: number;
+    /**
+     * Number of CPU threads to use (default2: 4).
+     */
+    threads?: number;
+    /**
+     * Enable verbose debug output from llama.cpp (default: false).
+     */
+    debug?: boolean;
+}
+export interface LlamaCppProvider {
+    (config: LlamaCppProviderConfig): LlamaCppLanguageModel;
+    languageModel(config: LlamaCppProviderConfig): LlamaCppLanguageModel;
+    embedding(config: LlamaCppProviderConfig): LlamaCppEmbeddingModel;
+}
+/**
+ * Creates a llama.cpp model provider.
+ *
+ * @example
+ * ```typescript
+ * import { llamaCpp } from 'ai-sdk-llama-cpp';
+ * import { embed, embedMany, generateText, streamText } from 'ai';
+ *
+ * const model = llamaCpp({
+ *   modelPath: './models/llama-3.2-1b.gguf'
+ * });
+ *
+ * const embeddingModel = llamaCpp.embedding({
+ *   modelPath: './models/nomic-embed-text-v1.5.Q4_K_M.gguf'
+ * });
+ *
+ * // Non-streaming
+ * const { text } = await generateText({
+ *   model,
+ *   prompt: 'Hello, how are you?'
+ * });
+ *
+ * // Streaming
+ * const { textStream } = await streamText({
+ *   model,
+ *   prompt: 'Tell me a story'
+ * });
+ *
+ * for await (const chunk of textStream) {
+ *   process.stdout.write(chunk);
+ * }
+ *
+ *
+ * // Single embedding
+ * const { embedding } = await embed({
+ *   model: embeddingModel,
+ *   value: 'Hello, world!'
+ * });
+ *
+ * // Multiple embeddings
+ * const { embeddings } = await embedMany({
+ *   model: embeddingModel,
+ *   values: ['Hello', 'World', 'How are you?']
+ * });
+ * ```
+ */
+export declare const llamaCpp: LlamaCppProvider;
+/**
+ * Default export for convenience.
+ */
+export default llamaCpp;
+//# sourceMappingURL=llama-cpp-provider.d.ts.map

package/dist/llama-cpp-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-cpp-provider.d.ts","sourceRoot":"","sources":["../src/llama-cpp-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,EAEtB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,sBAAsB,EACvB,MAAM,gCAAgC,CAAC;AAExC,MAAM,WAAW,sBAAsB;IACrC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAGD,MAAM,WAAW,gBAAgB;IAC/B,CAAC,MAAM,EAAE,sBAAsB,GAAG,qBAAqB,CAAC;IACxD,aAAa,CAAC,MAAM,EAAE,sBAAsB,GAAG,qBAAqB,CAAC;IACrE,SAAS,CAAC,MAAM,EAAE,sBAAsB,GAAG,sBAAsB,CAAC;CACnE;AA0BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,eAAO,MAAM,QAAQ,kBAAmB,CAAC;AAGzC;;GAEG;AACH,eAAe,QAAQ,CAAC"}

package/dist/llama-cpp-provider.js

@@ -0,0 +1,71 @@
+import { LlamaCppLanguageModel, } from "./llama-cpp-language-model.js";
+import { LlamaCppEmbeddingModel, } from "./llama-cpp-embedding-model.js";
+function createLlamaCpp() {
+    const provider = (config) => {
+        const modelConfig = {
+            modelPath: config.modelPath,
+            contextSize: config.contextSize,
+            gpuLayers: config.gpuLayers,
+            threads: config.threads,
+            debug: config.debug,
+        };
+        return new LlamaCppLanguageModel(modelConfig);
+    };
+    provider.languageModel = provider;
+    provider.embedding = (config) => {
+        return new LlamaCppEmbeddingModel(config);
+    };
+    return provider;
+}
+/**
+ * Creates a llama.cpp model provider.
+ *
+ * @example
+ * ```typescript
+ * import { llamaCpp } from 'ai-sdk-llama-cpp';
+ * import { embed, embedMany, generateText, streamText } from 'ai';
+ *
+ * const model = llamaCpp({
+ *   modelPath: './models/llama-3.2-1b.gguf'
+ * });
+ *
+ * const embeddingModel = llamaCpp.embedding({
+ *   modelPath: './models/nomic-embed-text-v1.5.Q4_K_M.gguf'
+ * });
+ *
+ * // Non-streaming
+ * const { text } = await generateText({
+ *   model,
+ *   prompt: 'Hello, how are you?'
+ * });
+ *
+ * // Streaming
+ * const { textStream } = await streamText({
+ *   model,
+ *   prompt: 'Tell me a story'
+ * });
+ *
+ * for await (const chunk of textStream) {
+ *   process.stdout.write(chunk);
+ * }
+ *
+ *
+ * // Single embedding
+ * const { embedding } = await embed({
+ *   model: embeddingModel,
+ *   value: 'Hello, world!'
+ * });
+ *
+ * // Multiple embeddings
+ * const { embeddings } = await embedMany({
+ *   model: embeddingModel,
+ *   values: ['Hello', 'World', 'How are you?']
+ * });
+ * ```
+ */
+export const llamaCpp = createLlamaCpp();
+/**
+ * Default export for convenience.
+ */
+export default llamaCpp;
+//# sourceMappingURL=llama-cpp-provider.js.map

package/dist/llama-cpp-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-cpp-provider.js","sourceRoot":"","sources":["../src/llama-cpp-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,GAEtB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,sBAAsB,GACvB,MAAM,gCAAgC,CAAC;AAqCxC,SAAS,cAAc;IACrB,MAAM,QAAQ,GAAG,CACf,MAA8B,EACP,EAAE;QACzB,MAAM,WAAW,GAAwB;YACvC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC;QAEF,OAAO,IAAI,qBAAqB,CAAC,WAAW,CAAC,CAAC;IAChD,CAAC,CAAA;IAED,QAAQ,CAAC,aAAa,GAAG,QAAQ,CAAC;IAElC,QAAQ,CAAC,SAAS,GAAG,CAAC,MAA8B,EAAE,EAAE;QACtD,OAAO,IAAI,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC5C,CAAC,CAAA;IAED,OAAO,QAA4B,CAAC;AACtC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;AAGzC;;GAEG;AACH,eAAe,QAAQ,CAAC"}

package/dist/native-binding.d.ts

@@ -0,0 +1,51 @@
+export interface LoadModelOptions {
+    modelPath: string;
+    gpuLayers?: number;
+    contextSize?: number;
+    threads?: number;
+    debug?: boolean;
+    /**
+     * Chat template to use for formatting messages.
+     * - "auto" (default): Use the template embedded in the GGUF model file
+     * - Template name: Use a specific built-in template (e.g., "llama3", "chatml", "gemma")
+     */
+    chatTemplate?: string;
+    /**
+     * Whether to load the model for embedding generation.
+     * When true, creates an embedding context with mean pooling enabled.
+     * Default: false
+     */
+    embedding?: boolean;
+}
+export interface ChatMessage {
+    role: string;
+    content: string;
+}
+export interface GenerateOptions {
+    messages: ChatMessage[];
+    maxTokens?: number;
+    temperature?: number;
+    topP?: number;
+    topK?: number;
+    stopSequences?: string[];
+}
+export interface GenerateResult {
+    text: string;
+    promptTokens: number;
+    completionTokens: number;
+    finishReason: 'stop' | 'length' | 'error';
+}
+export interface EmbedOptions {
+    texts: string[];
+}
+export interface EmbedResult {
+    embeddings: Float32Array[];
+    totalTokens: number;
+}
+export declare function loadModel(options: LoadModelOptions): Promise<number>;
+export declare function unloadModel(handle: number): boolean;
+export declare function generate(handle: number, options: GenerateOptions): Promise<GenerateResult>;
+export declare function generateStream(handle: number, options: GenerateOptions, onToken: (token: string) => void): Promise<GenerateResult>;
+export declare function isModelLoaded(handle: number): boolean;
+export declare function embed(handle: number, options: EmbedOptions): Promise<EmbedResult>;
+//# sourceMappingURL=native-binding.d.ts.map

package/dist/native-binding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"native-binding.d.ts","sourceRoot":"","sources":["../src/native-binding.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAiBD,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAYpE;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEnD;AAED,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAY1F;AAED,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,eAAe,EACxB,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAC/B,OAAO,CAAC,cAAc,CAAC,CAiBzB;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAErD;AAED,wBAAgB,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAYjF"}

package/dist/native-binding.js

@@ -0,0 +1,74 @@
+import { createRequire } from 'node:module';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
+// Load the native binding
+const binding = require(join(__dirname, '..', 'build', 'Release', 'llama_binding.node'));
+export function loadModel(options) {
+    return new Promise((resolve, reject) => {
+        binding.loadModel(options, (error, handle) => {
+            if (error) {
+                reject(new Error(error));
+            }
+            else if (handle !== null) {
+                resolve(handle);
+            }
+            else {
+                reject(new Error('Failed to load model: unknown error'));
+            }
+        });
+    });
+}
+export function unloadModel(handle) {
+    return binding.unloadModel(handle);
+}
+export function generate(handle, options) {
+    return new Promise((resolve, reject) => {
+        binding.generate(handle, options, (error, result) => {
+            if (error) {
+                reject(new Error(error));
+            }
+            else if (result) {
+                resolve(result);
+            }
+            else {
+                reject(new Error('Failed to generate: unknown error'));
+            }
+        });
+    });
+}
+export function generateStream(handle, options, onToken) {
+    return new Promise((resolve, reject) => {
+        binding.generateStream(handle, options, onToken, (error, result) => {
+            if (error) {
+                reject(new Error(error));
+            }
+            else if (result) {
+                resolve(result);
+            }
+            else {
+                reject(new Error('Failed to generate stream: unknown error'));
+            }
+        });
+    });
+}
+export function isModelLoaded(handle) {
+    return binding.isModelLoaded(handle);
+}
+export function embed(handle, options) {
+    return new Promise((resolve, reject) => {
+        binding.embed(handle, options, (error, result) => {
+            if (error) {
+                reject(new Error(error));
+            }
+            else if (result) {
+                resolve(result);
+            }
+            else {
+                reject(new Error('Failed to generate embeddings: unknown error'));
+            }
+        });
+    });
+}
+//# sourceMappingURL=native-binding.js.map

package/dist/native-binding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"native-binding.js","sourceRoot":"","sources":["../src/native-binding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,0BAA0B;AAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,oBAAoB,CAAC,CAAkB,CAAC;AAmE1G,MAAM,UAAU,SAAS,CAAC,OAAyB;IACjD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC3B,OAAO,CAAC,MAAM,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAAc;IACxC,OAAO,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,MAAc,EAAE,OAAwB;IAC/D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAClD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,MAAM,EAAE,CAAC;gBAClB,OAAO,CAAC,MAAM,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC;YACzD,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,MAAc,EACd,OAAwB,EACxB,OAAgC;IAEhC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,cAAc,CACpB,MAAM,EACN,OAAO,EACP,OAAO,EACP,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAChB,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,MAAM,EAAE,CAAC;gBAClB,OAAO,CAAC,MAAM,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAc;IAC1C,OAAO,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,MAAc,EAAE,OAAqB;IACzD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAC/C,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,MAAM,EAAE,CAAC;gBAClB,OAAO,CAAC,MAAM,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/native/CMakeLists.txt

@@ -0,0 +1,74 @@
+cmake_minimum_required(VERSION 3.15)
+project(llama_binding)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+# Build llama.cpp as a static library
+set(LLAMA_STATIC ON CACHE BOOL "Build llama.cpp as static library" FORCE)
+set(LLAMA_BUILD_TESTS OFF CACHE BOOL "Disable llama.cpp tests" FORCE)
+set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "Disable llama.cpp examples" FORCE)
+set(LLAMA_BUILD_SERVER OFF CACHE BOOL "Disable llama.cpp server" FORCE)
+
+# Enable Metal on macOS
+if(APPLE)
+    set(GGML_METAL ON CACHE BOOL "Enable Metal support" FORCE)
+    set(GGML_METAL_EMBED_LIBRARY ON CACHE BOOL "Embed Metal library" FORCE)
+endif()
+
+# Add llama.cpp subdirectory
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../llama.cpp ${CMAKE_CURRENT_BINARY_DIR}/llama.cpp)
+
+# Find Node.js and node-addon-api
+include_directories(${CMAKE_JS_INC})
+
+# Find node-addon-api
+execute_process(
+    COMMAND node -p "require('node-addon-api').include"
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE NODE_ADDON_API_DIR
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+string(REPLACE "\"" "" NODE_ADDON_API_DIR ${NODE_ADDON_API_DIR})
+include_directories(${NODE_ADDON_API_DIR})
+
+# Include llama.cpp headers
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../llama.cpp/include)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../llama.cpp/ggml/include)
+
+# Build the native addon
+add_library(${PROJECT_NAME} SHARED
+    binding.cpp
+    llama-wrapper.cpp
+)
+
+# Set output name and extension
+set_target_properties(${PROJECT_NAME} PROPERTIES
+    PREFIX ""
+    SUFFIX ".node"
+    OUTPUT_NAME "llama_binding"
+)
+
+# Link against llama.cpp
+target_link_libraries(${PROJECT_NAME} PRIVATE
+    llama
+    ggml
+    ${CMAKE_JS_LIB}
+)
+
+# Define NAPI_VERSION
+target_compile_definitions(${PROJECT_NAME} PRIVATE
+    NAPI_VERSION=8
+    NAPI_CPP_EXCEPTIONS
+)
+
+# Platform-specific settings
+if(APPLE)
+    target_link_libraries(${PROJECT_NAME} PRIVATE
+        "-framework Foundation"
+        "-framework Metal"
+        "-framework MetalKit"
+    )
+endif()
+