@shipworthy/ai-sdk-llama-cpp 0.2.0

package/CMakeLists.txt

@@ -0,0 +1,6 @@
+cmake_minimum_required(VERSION 3.15)
+project(llama_binding)
+
+# Include the native build
+add_subdirectory(native)
+

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Lars Grammel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,274 @@
+# ai-sdk-llama-cpp
+
+> **Alpha Software** - This package is in early development. The API may change between versions without notice.
+
+> **macOS Only** - This package currently only supports macOS with Apple Silicon or Intel processors.
+
+A minimal [llama.cpp](https://github.com/ggerganov/llama.cpp) provider for the [Vercel AI SDK](https://sdk.vercel.ai/), implementing the `LanguageModelV3` interface.
+
+This package loads llama.cpp directly into Node.js memory via native C++ bindings, enabling local LLM inference without requiring an external server.
+
+## Features
+
+- **Native Performance**: Direct C++ bindings using node-addon-api (N-API)
+- **GPU Acceleration**: Automatic Metal support on macOS
+- **Streaming & Non-streaming**: Full support for both `generateText` and `streamText`
+- **Chat Templates**: Automatic or configurable chat template formatting (llama3, chatml, gemma, etc.)
+- **ESM Only**: Modern ECMAScript modules, no CommonJS
+- **GGUF Support**: Load any GGUF-format model
+
+## Prerequisites
+
+Before installing, ensure you have the following:
+
+- **macOS** (Apple Silicon or Intel)
+- **Node.js** >= 18.0.0
+- **CMake** >= 3.15
+- **Xcode Command Line Tools**
+
+```bash
+# Install Xcode Command Line Tools (includes Clang)
+xcode-select --install
+
+# Install CMake via Homebrew
+brew install cmake
+```
+
+## Installation
+
+```bash
+npm install ai-sdk-llama-cpp
+```
+
+The installation will automatically:
+
+1. Detect macOS and verify platform compatibility
+2. Compile llama.cpp as a static library with Metal support
+3. Build the native Node.js addon
+
+> **Note**: Installation on Windows or Linux will fail with an error. Only macOS is supported.
+
+## Usage
+
+### Basic Example
+
+```typescript
+import { generateText } from "ai";
+import { llamaCpp } from "ai-sdk-llama-cpp";
+
+const model = llamaCpp({
+  modelPath: "./models/llama-3.2-1b-instruct.Q4_K_M.gguf",
+});
+
+try {
+  const { text } = await generateText({
+    model,
+    prompt: "Explain quantum computing in simple terms.",
+  });
+
+  console.log(text);
+} finally {
+  model.dispose();
+}
+```
+
+### Streaming Example
+
+```typescript
+import { streamText } from "ai";
+import { llamaCpp } from "ai-sdk-llama-cpp";
+
+const model = llamaCpp({
+  modelPath: "./models/llama-3.2-1b-instruct.Q4_K_M.gguf",
+});
+
+try {
+  const { textStream } = await streamText({
+    model,
+    prompt: "Write a haiku about programming.",
+  });
+
+  for await (const chunk of textStream) {
+    process.stdout.write(chunk);
+  }
+} finally {
+  model.dispose();
+}
+```
+
+### Embedding Example
+
+```typescript
+import { embed, embedMany } from "ai";
+import { llamaCpp } from "ai-sdk-llama-cpp";
+
+const model = llamaCpp.embedding({
+  modelPath: "./models/nomic-embed-text-v1.5.Q4_K_M.gguf",
+});
+
+try {
+  const { embedding } = await embed({
+    model,
+    value: "Hello, world!",
+  });
+
+  const { embeddings } = await embedMany({
+    model,
+    values: ["Hello, world!", "Hello, ▲!"],
+  });
+} finally {
+  model.dispose();
+}
+```
+
+### Configuration Options
+
+```typescript
+const model = llamaCpp({
+  // Required: Path to the GGUF model file
+  modelPath: "./models/your-model.gguf",
+
+  // Optional: Maximum context size (default: 2048)
+  contextSize: 4096,
+
+  // Optional: Number of layers to offload to GPU
+  // Default: 99 (all layers). Set to 0 to disable GPU.
+  gpuLayers: 99,
+
+  // Optional: Number of CPU threads (default: 4)
+  threads: 8,
+
+  // Optional: Enable verbose debug output from llama.cpp (default: false)
+  debug: true,
+
+  // Optional: Chat template to use for formatting messages
+  // - "auto" (default): Use the template embedded in the GGUF model file
+  // - Template name: Use a specific built-in template (e.g., "llama3", "chatml", "gemma")
+  chatTemplate: "auto",
+});
+```
+
+#### Chat Templates
+
+The `chatTemplate` option controls how messages are formatted before being sent to the model. Available templates include:
+
+- `chatml`, `llama2`, `llama2-sys`, `llama3`, `llama4`
+- `mistral-v1`, `mistral-v3`, `mistral-v7`
+- `phi3`, `phi4`, `gemma`, `falcon3`, `zephyr`
+- `deepseek`, `deepseek2`, `deepseek3`, `command-r`
+- And more (see llama.cpp documentation for the full list)
+
+### Generation Parameters
+
+The standard AI SDK generation parameters are supported:
+
+```typescript
+try {
+  const { text } = await generateText({
+    model,
+    prompt: "Hello!",
+    maxTokens: 256, // Maximum tokens to generate
+    temperature: 0.7, // Sampling temperature (0-2)
+    topP: 0.9, // Nucleus sampling threshold
+    topK: 40, // Top-k sampling
+    stopSequences: ["\n"], // Stop generation at these sequences
+  });
+} finally {
+  model.dispose();
+}
+```
+
+## Model Downloads
+
+You'll need to download GGUF-format models separately. Popular sources:
+
+- [Hugging Face](https://huggingface.co/models?search=gguf) - Search for GGUF models
+- [TheBloke's Models](https://huggingface.co/TheBloke) - Popular quantized models
+
+Example download:
+
+```bash
+# Create models directory
+mkdir -p models
+
+# Download a model (example: Llama 3.2 1B)
+wget -P models/ https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf
+```
+
+## API Reference
+
+### `llamaCpp(config)`
+
+Creates a new llama.cpp language model instance.
+
+**Parameters:**
+
+- `config.modelPath` (string, required): Path to the GGUF model file
+- `config.contextSize` (number, optional): Maximum context size. Default: 2048
+- `config.gpuLayers` (number, optional): GPU layers to offload. Default: 99
+- `config.threads` (number, optional): CPU threads. Default: 4
+- `config.debug` (boolean, optional): Enable verbose llama.cpp output. Default: false
+- `config.chatTemplate` (string, optional): Chat template to use for formatting messages. Default: "auto"
+
+**Returns:** `LlamaCppLanguageModel` - A language model compatible with the Vercel AI SDK
+
+### `LlamaCppLanguageModel`
+
+Implements the `LanguageModelV3` interface from `@ai-sdk/provider`.
+
+**Methods:**
+
+- `doGenerate(options)`: Non-streaming text generation
+- `doStream(options)`: Streaming text generation
+- `dispose()`: Unload the model and free GPU/CPU resources. **Always call this when done** to prevent memory leaks, especially when loading multiple models
+
+## Limitations
+
+This is a minimal implementation with the following limitations:
+
+- **macOS only**: Windows and Linux are not supported
+- **No tool/function calling**: Tool calls are not supported
+- **No image inputs**: Only text prompts are supported
+- **No JSON mode**: Structured output generation is not supported
+
+## Development
+
+### Building from Source
+
+```bash
+# Clone the repository
+git clone https://github.com/lgrammel/ai-sdk-llama-cpp.git
+cd ai-sdk-llama-cpp
+
+# Initialize submodules
+git submodule update --init --recursive
+
+# Install dependencies
+npm install
+
+# Build the native addon and TypeScript
+npm run build
+```
+
+### Scripts
+
+- `npm run build` - Build everything (native + TypeScript)
+- `npm run build:native` - Build only the native addon
+- `npm run build:ts` - Build only TypeScript
+- `npm run clean` - Remove build artifacts
+- `npm run test` - Run tests in watch mode
+- `npm run test:run` - Run all tests once
+- `npm run test:unit` - Run unit tests
+- `npm run test:integration` - Run integration tests
+- `npm run test:e2e` - Run end-to-end tests
+- `npm run test:coverage` - Run tests with coverage
+
+## License
+
+MIT
+
+## Acknowledgments
+
+- [llama.cpp](https://github.com/ggerganov/llama.cpp) - The underlying inference engine
+- [Vercel AI SDK](https://sdk.vercel.ai/) - The AI SDK framework
+- [node-addon-api](https://github.com/nodejs/node-addon-api) - N-API C++ wrapper

package/dist/binding-bun.d.ts

@@ -0,0 +1,7 @@
+import type { LoadModelOptions, GenerateOptions, GenerateResult } from './binding.js';
+export declare function loadModel(options: LoadModelOptions): Promise<number>;
+export declare function unloadModel(handle: number): boolean;
+export declare function isModelLoaded(handle: number): boolean;
+export declare function generate(handle: number, options: GenerateOptions): Promise<GenerateResult>;
+export declare function generateStream(handle: number, options: GenerateOptions, onToken: (token: string) => void): Promise<GenerateResult>;
+//# sourceMappingURL=binding-bun.d.ts.map

package/dist/binding-bun.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"binding-bun.d.ts","sourceRoot":"","sources":["../src/binding-bun.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AA2ItF,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAqDpE;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEnD;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAErD;AA6FD,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CA2C1F;AAED,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,eAAe,EACxB,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAC/B,OAAO,CAAC,cAAc,CAAC,CA4DzB"}

package/dist/binding-bun.js

@@ -0,0 +1,354 @@
+import { dlopen, FFIType, suffix, ptr, CString } from "bun:ffi";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+// Resolve the path to the shared library
+const libPath = join(__dirname, "..", "build", "Release", `libllama_ffi.${suffix}`);
+// Define FFI symbols for the C API
+const lib = dlopen(libPath, {
+    llama_load_model: {
+        args: [FFIType.ptr],
+        returns: FFIType.i32,
+    },
+    llama_unload_model: {
+        args: [FFIType.i32],
+        returns: FFIType.bool,
+    },
+    llama_is_model_loaded: {
+        args: [FFIType.i32],
+        returns: FFIType.bool,
+    },
+    llama_generate: {
+        args: [FFIType.i32, FFIType.ptr],
+        returns: FFIType.ptr,
+    },
+    llama_generate_stream: {
+        args: [FFIType.i32, FFIType.ptr, FFIType.ptr, FFIType.ptr],
+        returns: FFIType.ptr,
+    },
+    llama_free_result: {
+        args: [FFIType.ptr],
+        returns: FFIType.void,
+    },
+    llama_get_last_error: {
+        args: [],
+        returns: FFIType.ptr,
+    },
+    llama_clear_error: {
+        args: [],
+        returns: FFIType.void,
+    },
+});
+// Workaround for Bun FFI toArrayBuffer bug: https://github.com/oven-sh/bun/issues/23656
+// toArrayBuffer() always returns incorrect byteLength, so we use libc memcpy instead
+const libc = dlopen(process.platform === "darwin" ? "libc.dylib" : "libc.so.6", {
+    memcpy: {
+        args: [FFIType.ptr, FFIType.ptr, FFIType.u64],
+        returns: FFIType.ptr,
+    },
+});
+/**
+ * Copy memory from a native pointer to a JavaScript ArrayBuffer.
+ * This is a workaround for the Bun FFI toArrayBuffer bug.
+ * See: https://github.com/oven-sh/bun/issues/23656
+ */
+function copyFromCPtr(src, len) {
+    const out = new Uint8Array(len >>> 0);
+    libc.symbols.memcpy(ptr(out), src, len >>> 0);
+    return out.buffer;
+}
+// Helper to encode string to null-terminated buffer
+function encodeString(str) {
+    const encoder = new TextEncoder();
+    const encoded = encoder.encode(str);
+    const buffer = new Uint8Array(encoded.length + 1);
+    buffer.set(encoded);
+    buffer[encoded.length] = 0;
+    return buffer;
+}
+// Helper to read a null-terminated string from a pointer
+function readCString(pointer) {
+    const ptrValue = typeof pointer === "number" ? pointer : Number(pointer);
+    if (ptrValue === 0)
+        return null;
+    return new CString(pointer).toString();
+}
+// Structure sizes and offsets (for 64-bit systems with proper alignment)
+// llama_load_options_t layout:
+//   const char* model_path;      // offset 0, size 8
+//   int32_t gpu_layers;          // offset 8, size 4
+//   int32_t context_size;        // offset 12, size 4
+//   int32_t threads;             // offset 16, size 4
+//   bool debug;                  // offset 20, size 1
+//   padding                      // offset 21, size 3
+//   const char* chat_template;   // offset 24, size 8
+// Total: 32 bytes
+const LOAD_OPTIONS_SIZE = 32;
+// llama_generate_result_t layout:
+//   char* text;                  // offset 0, size 8
+//   int32_t prompt_tokens;       // offset 8, size 4
+//   int32_t completion_tokens;   // offset 12, size 4
+//   char* finish_reason;         // offset 16, size 8
+//   char* error;                 // offset 24, size 8
+// Total: 32 bytes
+const RESULT_TEXT_OFFSET = 0;
+const RESULT_PROMPT_TOKENS_OFFSET = 8;
+const RESULT_COMPLETION_TOKENS_OFFSET = 12;
+const RESULT_FINISH_REASON_OFFSET = 16;
+const RESULT_ERROR_OFFSET = 24;
+// llama_chat_message_t layout:
+//   const char* role;            // offset 0, size 8
+//   const char* content;         // offset 8, size 8
+// Total: 16 bytes
+const MESSAGE_SIZE = 16;
+// llama_generate_options_t layout:
+//   llama_chat_message_t* messages;  // offset 0, size 8
+//   size_t message_count;            // offset 8, size 8
+//   int32_t max_tokens;              // offset 16, size 4
+//   float temperature;               // offset 20, size 4
+//   float top_p;                     // offset 24, size 4
+//   int32_t top_k;                   // offset 28, size 4
+//   const char** stop_sequences;     // offset 32, size 8
+//   size_t stop_sequence_count;      // offset 40, size 8
+// Total: 48 bytes
+const GENERATE_OPTIONS_SIZE = 48;
+// Keep references to buffers to prevent GC
+const bufferRefs = new Map();
+let nextRefId = 1;
+function keepAlive(id, refs) {
+    bufferRefs.set(id, refs);
+}
+function releaseRefs(id) {
+    bufferRefs.delete(id);
+}
+export function loadModel(options) {
+    return new Promise((resolve, reject) => {
+        const refId = nextRefId++;
+        const refs = [];
+        try {
+            // Create load options structure
+            const optionsBuffer = new ArrayBuffer(LOAD_OPTIONS_SIZE);
+            const optionsView = new DataView(optionsBuffer);
+            // Encode strings
+            const modelPathBuffer = encodeString(options.modelPath);
+            refs.push(modelPathBuffer);
+            const chatTemplateBuffer = encodeString(options.chatTemplate ?? "auto");
+            refs.push(chatTemplateBuffer);
+            const modelPathPtr = ptr(modelPathBuffer);
+            const chatTemplatePtr = ptr(chatTemplateBuffer);
+            // Set pointers and values
+            optionsView.setBigUint64(0, BigInt(modelPathPtr), true); // model_path
+            optionsView.setInt32(8, options.gpuLayers ?? 99, true); // gpu_layers
+            optionsView.setInt32(12, options.contextSize ?? 2048, true); // context_size
+            optionsView.setInt32(16, options.threads ?? 4, true); // threads
+            optionsView.setUint8(20, options.debug ? 1 : 0); // debug
+            optionsView.setBigUint64(24, BigInt(chatTemplatePtr), true); // chat_template
+            refs.push(optionsBuffer);
+            keepAlive(refId, refs);
+            // Create a Uint8Array view that we keep a reference to
+            const optionsArray = new Uint8Array(optionsBuffer);
+            refs.push(optionsArray);
+            // Call the native function
+            const handle = lib.symbols.llama_load_model(ptr(optionsArray));
+            releaseRefs(refId);
+            if (handle < 0) {
+                const errorPtr = lib.symbols.llama_get_last_error();
+                const error = !isNullPtr(errorPtr) ? readCString(errorPtr) : "Failed to load model";
+                lib.symbols.llama_clear_error();
+                reject(new Error(error ?? "Failed to load model"));
+            }
+            else {
+                resolve(handle);
+            }
+        }
+        catch (error) {
+            releaseRefs(refId);
+            reject(error);
+        }
+    });
+}
+export function unloadModel(handle) {
+    return lib.symbols.llama_unload_model(handle);
+}
+export function isModelLoaded(handle) {
+    return lib.symbols.llama_is_model_loaded(handle);
+}
+function createGenerateOptions(options) {
+    const refs = [];
+    // Create message array
+    const messagesBuffer = new ArrayBuffer(MESSAGE_SIZE * options.messages.length);
+    const messagesView = new DataView(messagesBuffer);
+    refs.push(messagesBuffer);
+    for (let i = 0; i < options.messages.length; i++) {
+        const roleBuffer = encodeString(options.messages[i].role);
+        const contentBuffer = encodeString(options.messages[i].content);
+        refs.push(roleBuffer, contentBuffer);
+        const offset = i * MESSAGE_SIZE;
+        messagesView.setBigUint64(offset, BigInt(ptr(roleBuffer)), true);
+        messagesView.setBigUint64(offset + 8, BigInt(ptr(contentBuffer)), true);
+    }
+    // Create stop sequences array if provided
+    let stopSeqPtr = 0n;
+    if (options.stopSequences && options.stopSequences.length > 0) {
+        const stopSeqPtrBuffer = new ArrayBuffer(8 * options.stopSequences.length);
+        const stopSeqPtrView = new DataView(stopSeqPtrBuffer);
+        refs.push(stopSeqPtrBuffer);
+        for (let i = 0; i < options.stopSequences.length; i++) {
+            const seqBuffer = encodeString(options.stopSequences[i]);
+            refs.push(seqBuffer);
+            stopSeqPtrView.setBigUint64(i * 8, BigInt(ptr(seqBuffer)), true);
+        }
+        stopSeqPtr = BigInt(ptr(new Uint8Array(stopSeqPtrBuffer)));
+    }
+    // Create generate options structure
+    const optionsBuffer = new ArrayBuffer(GENERATE_OPTIONS_SIZE);
+    const optionsView = new DataView(optionsBuffer);
+    refs.push(optionsBuffer);
+    optionsView.setBigUint64(0, BigInt(ptr(new Uint8Array(messagesBuffer))), true); // messages
+    optionsView.setBigUint64(8, BigInt(options.messages.length), true); // message_count
+    optionsView.setInt32(16, options.maxTokens ?? 256, true); // max_tokens
+    optionsView.setFloat32(20, options.temperature ?? 0.7, true); // temperature
+    optionsView.setFloat32(24, options.topP ?? 0.9, true); // top_p
+    optionsView.setInt32(28, options.topK ?? 40, true); // top_k
+    optionsView.setBigUint64(32, stopSeqPtr, true); // stop_sequences
+    optionsView.setBigUint64(40, BigInt(options.stopSequences?.length ?? 0), true); // stop_sequence_count
+    return { buffer: optionsBuffer, refs };
+}
+// Helper to check if a pointer is null (can be null, 0, or 0n)
+function isNullPtr(ptr) {
+    if (ptr === null)
+        return true;
+    if (typeof ptr === "number")
+        return ptr === 0;
+    if (typeof ptr === "bigint")
+        return ptr === 0n;
+    // For Pointer type, convert to number and check
+    return Number(ptr) === 0;
+}
+function parseResult(resultPtr) {
+    // Get the numeric value of the pointer for null check
+    const ptrValue = typeof resultPtr === "number" ? resultPtr : Number(resultPtr);
+    if (ptrValue === 0) {
+        throw new Error("Generation failed: null result pointer");
+    }
+    const buffer = copyFromCPtr(resultPtr, 32);
+    const view = new DataView(buffer);
+    const textPtr = Number(view.getBigUint64(RESULT_TEXT_OFFSET, true));
+    const promptTokens = view.getInt32(RESULT_PROMPT_TOKENS_OFFSET, true);
+    const completionTokens = view.getInt32(RESULT_COMPLETION_TOKENS_OFFSET, true);
+    const finishReasonPtr = Number(view.getBigUint64(RESULT_FINISH_REASON_OFFSET, true));
+    const errorPtr = Number(view.getBigUint64(RESULT_ERROR_OFFSET, true));
+    const text = textPtr ? readCString(textPtr) ?? "" : "";
+    const finishReason = finishReasonPtr ? readCString(finishReasonPtr) ?? "error" : "error";
+    const error = errorPtr ? readCString(errorPtr) : null;
+    if (error) {
+        throw new Error(error);
+    }
+    return {
+        text,
+        promptTokens,
+        completionTokens,
+        finishReason: finishReason,
+    };
+}
+export function generate(handle, options) {
+    return new Promise((resolve, reject) => {
+        const refId = nextRefId++;
+        try {
+            const { buffer, refs } = createGenerateOptions(options);
+            keepAlive(refId, refs);
+            const resultPtr = lib.symbols.llama_generate(handle, ptr(new Uint8Array(buffer)));
+            if (isNullPtr(resultPtr)) {
+                releaseRefs(refId);
+                const errorPtr = lib.symbols.llama_get_last_error();
+                const error = !isNullPtr(errorPtr) ? readCString(errorPtr) : "Generation failed: null result";
+                lib.symbols.llama_clear_error();
+                reject(new Error(error ?? "Generation failed: null result"));
+                return;
+            }
+            try {
+                const result = parseResult(resultPtr);
+                resolve(result);
+            }
+            catch (parseError) {
+                // If parsing fails, check for library error before re-throwing
+                const errorPtr = lib.symbols.llama_get_last_error();
+                if (!isNullPtr(errorPtr)) {
+                    const libError = readCString(errorPtr);
+                    lib.symbols.llama_clear_error();
+                    if (libError) {
+                        reject(new Error(libError));
+                        return;
+                    }
+                }
+                throw parseError;
+            }
+            finally {
+                lib.symbols.llama_free_result(resultPtr);
+                releaseRefs(refId);
+            }
+        }
+        catch (error) {
+            releaseRefs(refId);
+            reject(error);
+        }
+    });
+}
+export function generateStream(handle, options, onToken) {
+    return new Promise((resolve, reject) => {
+        const refId = nextRefId++;
+        try {
+            const { buffer, refs } = createGenerateOptions(options);
+            keepAlive(refId, refs);
+            // Collect tokens during generation
+            // Note: Bun FFI callbacks are tricky, so we use a workaround
+            // The C API will call the callback synchronously during generation
+            // We store tokens and emit them after generation completes
+            // For now, we use the non-streaming generate and emit all at once
+            // A proper streaming implementation would require Bun's callback support
+            const resultPtr = lib.symbols.llama_generate(handle, ptr(new Uint8Array(buffer)));
+            if (isNullPtr(resultPtr)) {
+                releaseRefs(refId);
+                const errorPtr = lib.symbols.llama_get_last_error();
+                const error = !isNullPtr(errorPtr) ? readCString(errorPtr) : "Generation failed: null result";
+                lib.symbols.llama_clear_error();
+                reject(new Error(error ?? "Generation failed: null result"));
+                return;
+            }
+            try {
+                const result = parseResult(resultPtr);
+                // Emit the full text as tokens (character by character for now)
+                // In a proper implementation, we'd use the streaming API with callbacks
+                const words = result.text.split(/(\s+)/);
+                for (const word of words) {
+                    if (word) {
+                        onToken(word);
+                    }
+                }
+                resolve(result);
+            }
+            catch (parseError) {
+                // If parsing fails, check for library error before re-throwing
+                const errorPtr = lib.symbols.llama_get_last_error();
+                if (!isNullPtr(errorPtr)) {
+                    const libError = readCString(errorPtr);
+                    lib.symbols.llama_clear_error();
+                    if (libError) {
+                        reject(new Error(libError));
+                        return;
+                    }
+                }
+                throw parseError;
+            }
+            finally {
+                lib.symbols.llama_free_result(resultPtr);
+                releaseRefs(refId);
+            }
+        }
+        catch (error) {
+            releaseRefs(refId);
+            reject(error);
+        }
+    });
+}
+//# sourceMappingURL=binding-bun.js.map