@agorapete/wllama 3.5.1-q2.0

package/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "llama.cpp"]
+	path = llama.cpp
+	url = https://github.com/AgoraPete/llama.cpp.git

package/.prettierignore

@@ -0,0 +1,38 @@
+**/.vscode
+**/.github
+**/.git
+**/.svn
+**/.hg
+**/node_modules
+**/dist
+**/docs
+
+/llama.cpp
+
+/examples/advanced
+/examples/basic
+/examples/embeddings
+
+/scripts
+/esm
+/models
+/build
+
+/src/multi-thread
+/src/single-thread
+/src/wasm
+/src/workers-code/generated.ts
+/src/wasm-from-cdn.ts
+/src/glue/messages.ts
+
+/compat/wasm
+
+*.md
+*.mdx
+*.json
+*.lock
+*.yml
+*.cpp
+*.hpp
+
+*.config.js

package/AGENTS.md

@@ -0,0 +1 @@
+Refer to `README-dev.md` for development documentation.

package/CMakeLists.txt

@@ -0,0 +1,131 @@
+cmake_minimum_required(VERSION 3.14)
+project("wllama")
+
+option(WLLAMA_TEST_BACKEND "Build wllama with test-backend-ops included" OFF)
+
+set(MTMD_VIDEO OFF CACHE BOOL "" FORCE)
+
+set(CMAKE_THREAD_LIBS_INIT "-lpthread")
+set(CMAKE_HAVE_THREADS_LIBRARY 1)
+set(CMAKE_USE_WIN32_THREADS_INIT 0)
+set(CMAKE_USE_PTHREADS_INIT 1)
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+
+set(WLLAMA_COMPILE_OPTIONS
+    -O3 -msimd128 -DNDEBUG
+    # -flto=full # note: this breaks things, better to disable
+    -frtti
+    -pthread
+    -gsource-map
+)
+set(WLLAMA_LINK_OPTIONS
+    # -flto=full # note: this breaks things, better to disable
+    --no-entry
+    -sEXPORT_ES6=0
+    -sMODULARIZE=0
+    -sINITIAL_MEMORY=128MB
+    -sMAXIMUM_MEMORY=4096MB
+    -sSTACK_SIZE=5MB
+    -sALLOW_MEMORY_GROWTH=1
+    -sFORCE_FILESYSTEM=1
+    -sEXPORTED_FUNCTIONS=_main,_wllama_malloc,_wllama_start,_wllama_action,_wllama_exit,_wllama_debug
+    -sEXPORTED_RUNTIME_METHODS=ccall,cwrap,HEAPU8,MEMFS,FS,mmapAlloc,ENV,wasmMemory
+    -sNO_EXIT_RUNTIME=1
+    -sIMPORTED_MEMORY=1
+    -sPTHREAD_POOL_SIZE=Module[\"pthreadPoolSize\"]
+    -sUSE_PTHREADS=1
+    -pthread
+    -gsource-map
+    --emit-symbol-map
+    -Wl,--wrap,fopen
+    -Wl,--wrap,fclose
+    -Wl,--wrap,fread
+    -Wl,--wrap,fseek
+    -Wl,--wrap,ftell
+    -Wl,--wrap,abort
+    -Wl,--wrap,ggml_graph_plan # for test-backend-ops
+)
+
+if (WLLAMA_COMPAT)
+    # no wasm exception (not compatible with asyncify - asyncify is needed for firefox and safari)
+    # no mem64 (not compatible with safari)
+    list(APPEND WLLAMA_COMPILE_OPTIONS
+        -fexceptions
+        -pthread
+    )
+    list(APPEND WLLAMA_LINK_OPTIONS
+        -fexceptions
+        -sASYNCIFY=1
+        -sASYNCIFY_ADD=['wllama_start','wllama_action']
+    )
+else()
+    list(APPEND WLLAMA_COMPILE_OPTIONS
+        -sMEMORY64=1
+        -fwasm-exceptions
+    )
+    list(APPEND WLLAMA_LINK_OPTIONS
+        -sMEMORY64=1
+        -fwasm-exceptions
+        -sJSPI
+        -sJSPI_EXPORTS=['wllama_start','wllama_action']
+    )
+endif()
+
+add_compile_options(${WLLAMA_COMPILE_OPTIONS})
+add_link_options(${WLLAMA_LINK_OPTIONS})
+
+add_subdirectory(llama.cpp)
+
+set(LLAMA_INSTALL_VERSION 0.0.${LLAMA_BUILD_NUMBER})
+add_subdirectory(llama.cpp/tools/mtmd)
+
+file(GLOB_RECURSE LLAMA_COMMON_SRC
+    CONFIGURE_DEPENDS
+    llama.cpp/common/*.cpp
+    llama.cpp/common/*.h
+    llama.cpp/common/*.hpp
+    llama.cpp/common/jinja/*.cpp
+    llama.cpp/common/jinja/*.h
+)
+
+list(REMOVE_ITEM LLAMA_COMMON_SRC
+    ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/common/arg.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/common/console.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/common/debug.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/common/fit.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/common/hf-cache.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/common/log.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/common/preset.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/common/download.cpp
+)
+
+set(LLAMA_SERVER_SRC llama.cpp/tools/server/server-context.cpp
+    llama.cpp/tools/server/server-task.cpp
+    llama.cpp/tools/server/server-chat.cpp
+    llama.cpp/tools/server/server-common.cpp
+)
+
+set(WLLAMA_SRC ${LLAMA_COMMON_SRC} ${LLAMA_SERVER_SRC}
+    cpp/wllama.cpp
+    cpp/glue.hpp
+    llama.cpp/include/llama.h)
+
+if(WLLAMA_TEST_BACKEND)
+    message(WARNING "Building wllama with test-backend-ops included")
+    set(TEST_BACKEND_OPS_SRC ${CMAKE_CURRENT_BINARY_DIR}/test-backend-ops.cpp)
+    file(READ ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/tests/test-backend-ops.cpp TEST_BACKEND_OPS_CONTENT)
+    string(REPLACE "int main(" "int main_test_backend_ops(" TEST_BACKEND_OPS_CONTENT "${TEST_BACKEND_OPS_CONTENT}")
+    file(WRITE ${TEST_BACKEND_OPS_SRC} "${TEST_BACKEND_OPS_CONTENT}")
+    list(APPEND WLLAMA_SRC ${TEST_BACKEND_OPS_SRC})
+    add_compile_definitions(WLLAMA_TEST_BACKEND)
+endif()
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/cpp)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/include)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/common)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/vendor)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/tools/server)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/tools/mtmd)
+
+add_executable(wllama ${WLLAMA_SRC})
+target_link_libraries(wllama PRIVATE ggml llama mtmd ${CMAKE_THREAD_LIBS_INIT})

package/LICENCE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Xuan Son NGUYEN
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README-dev.md

@@ -0,0 +1,178 @@
+# wllama
+
+Wllama is a webassembly binding of llama.cpp. It contains the main source code of llama.cpp compiled to wasm (with emscripten), plus a wrapper to provide various convenient APIs, including: downloading and caching models, compatibility, etc.
+
+## Project structure
+
+The project has these directories:
+- `src`: the main typescript source code
+- `cpp`: C++ interface
+- `scripts`: various scripts for development
+- `examples`: various examples
+
+The project has these main components:
+- `wllama.ts`: the main public API
+- `model-manager.ts`: relies on cache manager to manage models. For example, a model can be composed of multiple files
+    - `cache-manager.ts`: interface for managing cache files. It uses OPFS under the hood
+    - `huggingface.ts`: utility for managing models downloading from hugging face hub
+- `worker.ts`: the worker manager that will be responsible of starting the emscripten worker and maintaining the communication with it
+- `glue.ts`: GLUE implementation
+- `wllama.cpp`: the main C++ interface
+
+### GLUE
+
+GLUE is a home-grown binary protocol inspired by Protobuf. It is used internally to communicate between the wasm context and the JavaScript context of wllama.
+
+The main goal of GLUE is to allow a type-safe interface with low overhead. It works by serializing messages into `ArrayBuffer` and transferring them using [Transferable objects](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Transferable_objects), which avoids copying.
+
+**Wire format:**
+- 4 bytes - magic number (`GLUE`)
+- 4 bytes - version number (`GLUE_VERSION`)
+- 8 bytes - message prototype ID
+- 4 bytes - message length (unsigned)
+- message fields, each encoded as:
+  - 4 bytes data type (e.g. `int`, `float`, `str`, `raw`, and array variants)
+  - 4 bytes size (only for arrays and strings)
+  - data bytes
+
+**Supported field types:** `str`, `int`, `float`, `bool`, `raw` (arbitrary bytes), and array variants of each.
+
+Upon build, `generate_glue_prototype.js` reads `glue.hpp` and generates `glue/messages.ts`, which provides the TypeScript-side message types used throughout the codebase.
+
+### Threading model
+
+Wllama ships a **single wasm build** that supports both single-threaded and multi-threaded execution. The number of threads is determined at runtime rather than at compile time.
+
+At startup, wllama checks whether the browser supports `SharedArrayBuffer` (required for wasm threads). This check validates both the existence of `SharedArrayBuffer` and whether the wasm atomics feature is available (COOP/COEP headers must be set by the server for `SharedArrayBuffer` to be accessible).
+
+The thread pool size is passed to emscripten via `-sPTHREAD_POOL_SIZE=Module["pthreadPoolSize"]`:
+- If the browser **supports** shared memory: `pthreadPoolSize` is set to the desired thread count (defaults to `hardwareConcurrency / 2`)
+- If the browser **does not support** shared memory: `pthreadPoolSize` is set to `0`, which disables pthreads entirely and falls back to single-threaded execution
+
+This logic lives in `wllama.ts` (`isSupportMultiThread()` from `utils.ts` performs the feature detection).
+
+## Startup process
+
+Upon startup, these steps are performed:
+- `ProxyToWorker` is created in the main wllama JS context
+- A web worker is spawned, the code is taken from `workers-code/generated.ts`
+- The worker loads emscripten code, sets up the environment then eventually calls the `main()` inside `wllama.cpp`. These preparation steps are injected (see `llama-cpp.js`):
+    - Hooking `printf` functions
+    - Setting up HeapFS
+    - Setting up communication callbacks
+
+## File access
+
+Wllama employs some tricks to avoid making copies while reading GGUF files. The runtime uses one of these 2 mechanisms. See `workers-code/llama-cpp.js` for the implementation.
+
+Please note that wllama only accepts `Blob` as input data.
+
+### Async file read
+
+This implementation hooks into `fopen`, `fseek` and `fread`, and forwards these calls to the main thread (via message port), where we eventually call `Blob.slice()` to read the data. Because of the asynchronous execution via `onmessage` and `postMessage`, JSPI / Asyncify is required.
+
+Upon running, action `fs.alloc` is fired to indicate that the file can be read through JSPI / Asyncify call. The actual buffer won't be allocated for the file, but only the metadata is.
+
+When wasm calls `fread()`:
+- `fread()` calls `await fileRead()` in the JS context
+- `fileRead()` posts a message of type `fs.read_req` to the main thread
+- Main thread uses `Blob.slice()` to read the data, then sends it back via a `fs.read_res` message
+- Worker's `onmessage` receives the message and resumes the awaiting coroutine
+
+Note:
+- While awaiting the read data, the worker should not have any other activities (a global variable is used as a guard and will raise an exception on any incoming messages)
+- The minimum read size is 1MB. If less than this amount is requested, the full 1MB block is cached for subsequent reads. This is because reading GGUF metadata frequently involves reads of less than 1KB at a time, which can become a bottleneck without caching.
+- Env var `USE_ASYNC_FILE` is used to signal from JS to wasm that we are using async file read (upon starting the module). If `USE_ASYNC_FILE` is not set, we fallback to HeapFS/mmap case (see in next section)
+
+### HeapFS
+
+HeapFS is a lightweight wrapper around emscripten's default FS driver. The main goal is to allow `mmap()` to map to existing data instead of copying it (the default emscripten behavior).
+
+These steps are performed:
+
+- Action `fs.alloc` is fired to create the file handle and file buffer in the wasm context
+- The main thread then creates and holds a `ReadableStream` for the `Blob`
+- The main thread reads the file chunk by chunk, streaming it to the worker via `fs.write` messages
+- Once streaming is finished, the `ReadableStream` is closed
+- The model load is then triggered with `mmap = true`, and `mmap()` is wrapped to return a pointer to the correct data in the buffer allocated in step 1
+
+The main downside of this approach is that on WebGPU, even though some tensors can be offloaded to the GPU, we still need to allocate the full model in main memory. For example, a 4GB model will still occupy 4GB of main memory, even if half of the layers (~2GB) are offloaded to the GPU.
+
+## Compressed source map
+
+Emscripten's `--emit-symbol-map` flag produces a `.js.symbols` file mapping each wasm function index to its demangled C++ name. `scripts/build_source_map.js` reads this file alongside the `.wasm` binary and produces a single TypeScript file (`src/wasm/source-map.ts`) containing a compact deduplicated name table per build, gzip-compressed and base64-encoded.
+
+The script runs automatically as part of the docker build (see `scripts/docker-compose.yml`). It can also be run manually:
+
+```sh
+# uses build/ and build-compat/ by default
+node scripts/build_source_map.js
+
+# or with explicit paths
+node scripts/build_source_map.js \
+  --input default:build \
+  --input compat:build-compat \
+  --output src/wasm/source-map.ts
+```
+
+### Name cleaning rules
+
+Raw demangled names can be hundreds of characters. The following rules are applied in order:
+
+1. **std:: collapse** - any name starting with `std::` is replaced with the single hint `std::...`
+2. **Lambda/closure extraction** - names containing `::$_N` or `::'lambda'` are replaced with the nearest enclosing context (the segment inside the last `<…>` before the marker)
+3. **Parameter stripping** - parameter lists are dropped; empty `()` is kept, non-empty is removed entirely
+4. **libc++ internals** - `::__1::`, `::__2::`, etc. are collapsed to `::`
+5. **ABI tags** - `[abi:…]` annotations are removed
+6. **Template truncation** - template argument content longer than 10 characters is truncated to `<first10chars...>`
+7. **Final cleanup** - double `::::` collapsed, whitespace normalised
+
+### Binary format (before gzip)
+
+All integers are little-endian.
+
+```
+┌──────────────────────────────────────────────────────────┐
+│ HEADER (12 bytes)                                        │
+│   u32  first_func_id  - wasm function index of entry 0  │
+│   u32  num_funcs      - number of functions              │
+│   u32  num_names      - number of unique names           │
+├──────────────────────────────────────────────────────────┤
+│ NAME TABLE  (num_names entries)                          │
+│   for each name:                                         │
+│     u8   length       - byte length of name (max 254)   │
+│     u8[] name         - UTF-8 string (no null term)      │
+├──────────────────────────────────────────────────────────┤
+│ INDEX ARRAY  (num_funcs × u16)                           │
+│   u16  name_idx       - index into name table            │
+│                         0xFFFF = no name / unknown       │
+└──────────────────────────────────────────────────────────┘
+```
+
+To decode at runtime: base64-decode -> `DecompressionStream('gzip')` -> parse binary. Given a wasm function index `id`, look up `index_array[id - first_func_id]` to get the name table slot.
+%
+## Debugging backend ops
+
+> [!IMPORTANT]
+>
+> By default, the build does NOT include `test-backend-ops` to save space. If you need to run it, please clone the repo and build it yourself, instructions below
+
+Requirements:
+- You have Docker installed and running on your machine
+- On Windows, please use WSL
+
+1. Clone this repo locally: `git clone --recurse-submodules https://github.com/ngxson/wllama.git`
+2. `npm run build:test && npm run build`
+3. `npm run serve` and open http://localhost:8080/examples/test-backend-ops/
+
+Note: A debugging build cannot be merged to `master` or publish to npm
+
+## Build process
+
+The build process uses emscripten in docker to compile the project.
+
+After compilation, `generate_glue_prototype.js` is called to generate the GLUE message types to be used in TypeScript.
+
+Built wasm file will then be copied to the `src` directory.
+
+Finally, `build_worker.sh` is called to generate the web worker code.

package/README.md

@@ -0,0 +1,225 @@
+# wllama - Wasm binding for llama.cpp
+
+![](./README_banner.png)
+
+WebAssembly binding for [llama.cpp](https://github.com/ggerganov/llama.cpp)
+
+👉 [Try the demo app](https://huggingface.co/spaces/ngxson/wllama)
+
+👉 See the [blog post](https://reeselevine.github.io/llamas-on-the-web/) introducing WebGPU support in llama.cpp and wllama
+
+📄 [Documentation](https://github.ngxson.com/wllama/docs/)
+
+For changelog, please visit [releases page](https://github.com/ngxson/wllama/releases)
+
+> [!IMPORTANT]
+>
+> **🔥🔥 V3 is out, with WebGPU, multimodal and tool calling support. Read the [V3 release guide](./guides/intro-v3.md)**
+>
+> For compatibility issues, please refer to [@wllama/wllama-compat](./compat/README.md)
+
+![](./assets/screenshot_0.png)
+
+## Features
+
+- 🔌 OpenAI-compatible API (fully-typed built-in)
+- 🚀 WebGPU support
+- 🔥 Multimodal support (image and audio file input)
+- 🔥 Tool calling support
+- Can run inference directly on browser (using [WebAssembly SIMD](https://emscripten.org/docs/porting/simd.html)), no backend or GPU is needed!
+- No runtime dependency (see [package.json](./package.json))
+- Ability to split the model into smaller files and load them in parallel (same as `split` and `cat`)
+- Auto switch between single-thread and multi-thread build based on browser support
+- Inference is done inside a worker, does not block UI render
+- Pre-built npm package [@wllama/wllama](https://www.npmjs.com/package/@wllama/wllama)
+
+Limitations:
+- To enable multi-thread, you must add `Cross-Origin-Embedder-Policy` and `Cross-Origin-Opener-Policy` headers. See [this discussion](https://github.com/ffmpegwasm/ffmpeg.wasm/issues/106#issuecomment-913450724) for more details.
+- Max file size is 2GB, due to [size restriction of ArrayBuffer](https://stackoverflow.com/questions/17823225/do-arraybuffers-have-a-maximum-length). If your model is bigger than 2GB, please follow the **Split model** section below.
+
+## Code demo and documentation
+
+Demo:
+- Basic usages with completions and embeddings: https://github.ngxson.com/wllama/examples/basic/ ([source code](./examples/basic/index.html))
+- Embedding and cosine distance: https://github.ngxson.com/wllama/examples/embeddings/ ([source code](./examples/embeddings/index.html))
+- Multimodal (vision) completion: https://github.ngxson.com/wllama/examples/multimodal/ ([source code](./examples/multimodal/index.html))
+- Tool calling: https://github.ngxson.com/wllama/examples/tools/ ([source code](./examples/tools/index.html))
+
+## How to use
+
+### Use Wllama inside React Typescript project
+
+Install it:
+
+```bash
+npm i @wllama/wllama
+```
+
+Then, import the module:
+
+```ts
+import { Wllama } from '@wllama/wllama';
+let wllamaInstance = new Wllama(WLLAMA_CONFIG_PATHS, ...);
+// (the rest is the same with earlier example)
+```
+
+For complete code example, see [examples/main/src/utils/wllama.context.tsx](./examples/main/src/utils/wllama.context.tsx)
+
+NOTE: this example only covers completions usage. For embeddings, please see [examples/embeddings/index.html](./examples/embeddings/index.html)
+
+### WebGPU support
+
+WebGPU support is introduced via [PR #215](https://github.com/ngxson/wllama/pull/215).
+
+Upon updating to V3.1, WebGPU will be enabled automatically. By default, all layers will be offloaded to GPU. If the model is too big to fit into VRAM, you can manually adjust the number of layers via the `n_gpu_layers` parameter of `LoadModelParams`. Example:
+
+```js
+// (optionally) will allow running WebGPU on Firefox via compat mode; performance will be significantly degraded
+wllama.setCompat('default', 'firefox_safari');
+
+await wllama.loadModel(files, {
+  n_gpu_layers: 4, // meaning 4 layers are offloaded to GPU; set to 0 to disable GPU inference
+});
+```
+
+### Prepare your model
+
+- It is recommended to split the model into **chunks of maximum 512MB**. This will result in slightly faster download speed (because multiple splits can be downloaded in parallel), and also prevent some out-of-memory issues. **See the "Split model" section below for more details.**
+- It is recommended to use quantized Q4, Q5 or Q6 for balance among performance, file size and quality. Using IQ (with imatrix) is **not** recommended, may result in slow inference and low quality.
+
+### Simple usage with ES6 module
+
+For complete code, see [examples/basic/index.html](./examples/basic/index.html)
+
+```javascript
+import { Wllama } from './esm/index.js';
+
+(async () => {
+  const CONFIG_PATHS = {
+    default: './esm/wasm/wllama.wasm',
+  };
+  // Automatically switch between single-thread and multi-thread version based on browser support
+  // If you want to enforce single-thread, add { "n_threads": 1 } to LoadModelConfig
+  const wllama = new Wllama(CONFIG_PATHS);
+  // Define a function for tracking the model download progress
+  const progressCallback =  ({ loaded, total }) => {
+    // Calculate the progress as a percentage
+    const progressPercentage = Math.round((loaded / total) * 100);
+    // Log the progress in a user-friendly format
+    console.log(`Downloading... ${progressPercentage}%`);
+  };
+  // Load GGUF from Hugging Face hub
+  // (alternatively, you can use loadModelFromUrl if the model is not from HF hub)
+  await wllama.loadModelFromHF(
+    { repo: 'ggml-org/models', file: 'tinyllamas/stories260K.gguf' },
+    { progressCallback }
+  );
+  const response = await wllama.createChatCompletion({
+    messages: [{ role: 'user', content: elemInput.value }],
+    max_tokens: 50,
+    temperature: 0.5,
+    top_k: 40,
+    top_p: 0.9,
+  });
+  console.log(response.choices[0].message.content);
+})();
+```
+
+Alternatively, you can use the `*.wasm` files from CDN:
+
+```js
+import WasmFromCDN from '@wllama/wllama/esm/wasm-from-cdn.js';
+const wllama = new Wllama(WasmFromCDN);
+// NOTE: this is not recommended, only use when you can't embed wasm files in your project
+```
+
+### Split model
+
+Cases where we want to split the model:
+- Due to [size restriction of ArrayBuffer](https://stackoverflow.com/questions/17823225/do-arraybuffers-have-a-maximum-length), the size limitation of a file is 2GB. If your model is bigger than 2GB, you can split the model into small files.
+- Even with a small model, splitting into chunks allows the browser to download multiple chunks in parallel, thus making the download process a bit faster.
+
+We use `llama-gguf-split` to split a big gguf file into smaller files. You can download the pre-built binary via [llama.cpp release page](https://github.com/ggerganov/llama.cpp/releases):
+
+```bash
+# Split the model into chunks of 512 Megabytes
+./llama-gguf-split --split-max-size 512M ./my_model.gguf ./my_model
+```
+
+This will output files ending with `-00001-of-00003.gguf`, `-00002-of-00003.gguf`, and so on.
+
+You can then pass to `loadModelFromUrl` or `loadModelFromHF` the URL of the first file and it will automatically load all the chunks:
+
+```js
+const wllama = new Wllama(CONFIG_PATHS, {
+  parallelDownloads: 5, // optional: maximum files to download in parallel (default: 3)
+});
+await wllama.loadModelFromHF({
+  repo: 'ngxson/tinyllama_split_test',
+  file: 'stories15M-q8_0-00001-of-00003.gguf',
+});
+```
+
+### Custom logger (suppress debug messages)
+
+When initializing Wllama, you can pass a custom logger to Wllama.
+
+Example 1: Suppress debug message
+
+```js
+import { Wllama, LoggerWithoutDebug } from '@wllama/wllama';
+
+const wllama = new Wllama(pathConfig, {
+  // LoggerWithoutDebug is predefined inside wllama
+  logger: LoggerWithoutDebug,
+});
+```
+
+Example 2: Add emoji prefix to log messages
+
+```js
+const wllama = new Wllama(pathConfig, {
+  logger: {
+    debug: (...args) => console.debug('🔧', ...args),
+    log: (...args) => console.log('ℹ️', ...args),
+    warn: (...args) => console.warn('⚠️', ...args),
+    error: (...args) => console.error('☠️', ...args),
+  },
+});
+```
+
+## How to compile the binary yourself
+
+This repository already come with pre-built binary from llama.cpp source code. However, in some cases you may want to compile it yourself:
+- You don't trust the pre-built one.
+- You want to try out latest - bleeding-edge changes from upstream llama.cpp source code.
+
+You can use the commands below to compile it yourself:
+
+```shell
+# /!\ IMPORTANT: Require having docker compose installed
+
+# Clone the repository with submodule
+git clone --recurse-submodules https://github.com/ngxson/wllama.git
+cd wllama
+
+# Optionally, you can run this command to update llama.cpp to latest upstream version (bleeding-edge, use with your own risk!)
+# git submodule update --remote --merge
+
+# Install the required modules
+npm i
+
+# Firstly, build llama.cpp into wasm
+npm run build:wasm
+# Then, build ES module
+npm run build
+```
+
+## TODO
+
+- Add support for LoRA adapter
+- Support multi-sequences: knowing the resource limitation when using WASM, I don't think having multi-sequences is a good idea
+
+## Acknowledgments
+
+Wllama was created and is maintained by [Xuan-Son Nguyen](https://ngxson.com/). The WebGPU backend for llama.cpp is maintained by [Reese Levine](https://reeselevine.github.io/). We thank all other contributors to both wllama and llama.cpp, whose work made this project possible.