@agorapete/wllama 3.5.1-q2.0

package/cpp/wllama-fs.h

@@ -0,0 +1,148 @@
+#pragma once
+
+#ifdef __EMSCRIPTEN__
+#include <emscripten/emscripten.h>
+#endif
+
+#include <algorithm>
+#include <map>
+#include <vector>
+#include <cstring>
+
+static std::map<FILE *, std::string> s_file_path_map;
+
+namespace wllama_fs
+{
+  bool ready = false;
+  bool use_async = false;
+
+  static const size_t CACHE_SIZE = 1024 * 1024; // 1 MB read-ahead
+
+  std::vector<uint8_t> cache_data;
+  size_t cache_start = 0;
+  FILE *cache_file = nullptr;
+
+  void make_sure_ready()
+  {
+    if (ready)
+      return;
+    use_async = getenv("USE_ASYNC_FILE") != nullptr;
+    ready = true;
+  }
+
+  size_t try_cache(FILE *f, char *ptr, size_t req_bytes, size_t fpos)
+  {
+    if (f != cache_file || cache_data.empty())
+      return 0;
+    if (fpos >= cache_start && fpos + req_bytes <= cache_start + cache_data.size())
+    {
+      memcpy(ptr, cache_data.data() + (fpos - cache_start), req_bytes);
+      return req_bytes;
+    }
+    return 0;
+  }
+}
+
+// Thin stub — real implementation lives in llama-cpp.js to avoid
+// C++ formatter mangling the JS syntax inside EM_ASYNC_JS macros.
+
+EM_ASYNC_JS(size_t, js_file_read, (const char *path_ptr, size_t offset, size_t req_size, void *out_ptr), {
+  return await _wllama_js_file_read(UTF8ToString(Number(path_ptr)), Number(offset), Number(req_size), Number(out_ptr));
+});
+
+extern "C"
+{
+  FILE *__real_fopen(const char *path, const char *mode);
+  int __real_fclose(FILE *f);
+  size_t __real_fread(void *ptr, size_t size, size_t nmemb, FILE *f);
+  int __real_fseek(FILE *f, long offset, int whence);
+  long __real_ftell(FILE *f);
+
+  FILE *__wrap_fopen(const char *path, const char *mode)
+  {
+    wllama_fs::make_sure_ready();
+    FILE *f = __real_fopen(path, mode);
+    if (f)
+    {
+      s_file_path_map[f] = path;
+    }
+    return f;
+  }
+
+  int __wrap_fclose(FILE *f)
+  {
+    if (wllama_fs::cache_file == f)
+    {
+      wllama_fs::cache_file = nullptr;
+      wllama_fs::cache_data.clear();
+    }
+    s_file_path_map.erase(f);
+    return __real_fclose(f);
+  }
+
+  int __wrap_fseek(FILE *f, long offset, int whence)
+  {
+    return __real_fseek(f, offset, whence);
+  }
+
+  long __wrap_ftell(FILE *f)
+  {
+    return __real_ftell(f);
+  }
+
+  size_t __wrap_fread(void *ptr, size_t size, size_t nmemb, FILE *f)
+  {
+    wllama_fs::make_sure_ready();
+    if (!wllama_fs::use_async)
+      return __real_fread(ptr, size, nmemb, f);
+
+    auto nit = s_file_path_map.find(f);
+    if (nit == s_file_path_map.end())
+      return __real_fread(ptr, size, nmemb, f);
+
+    size_t req_bytes = size * nmemb;
+    if (req_bytes == 0)
+      return 0;
+
+    size_t fpos = (size_t)__real_ftell(f);
+
+    // Large reads (>= 1 MB): write directly into ptr, skip cache entirely.
+    if (req_bytes >= wllama_fs::CACHE_SIZE)
+    {
+      size_t actual = (size_t)js_file_read(
+          nit->second.c_str(), fpos, req_bytes, ptr);
+      if (actual == 0)
+        return 0;
+      size_t copy_bytes = std::min(req_bytes, actual);
+      __real_fseek(f, fpos + copy_bytes, SEEK_SET);
+      return copy_bytes / size;
+    }
+
+    // Small reads: try cache first.
+    size_t cached = wllama_fs::try_cache(f, (char *)ptr, req_bytes, fpos);
+    if (cached == req_bytes)
+    {
+      __real_fseek(f, fpos + req_bytes, SEEK_SET);
+      return nmemb;
+    }
+
+    // Cache miss: fetch a full CACHE_SIZE block from main thread.
+    wllama_fs::cache_data.resize(wllama_fs::CACHE_SIZE);
+    size_t actual = (size_t)js_file_read(
+        nit->second.c_str(), fpos, wllama_fs::CACHE_SIZE,
+        wllama_fs::cache_data.data());
+
+    wllama_fs::cache_data.resize(actual);
+    wllama_fs::cache_file  = f;
+    wllama_fs::cache_start = fpos;
+
+    if (actual == 0)
+      return 0;
+
+    size_t copy_bytes = std::min(req_bytes, actual);
+    memcpy(ptr, wllama_fs::cache_data.data(), copy_bytes);
+    __real_fseek(f, fpos + copy_bytes, SEEK_SET);
+
+    return copy_bytes / size;
+  }
+}

package/cpp/wllama.cpp

@@ -0,0 +1,187 @@
+#include <iostream>
+#include <vector>
+#include <string>
+#include <sstream>
+#include <stdio.h>
+
+#include <stdlib.h>
+#include <unistd.h>
+
+#ifdef __EMSCRIPTEN__
+#include <malloc.h>
+#include <emscripten/emscripten.h>
+#endif
+
+// #define GLUE_DEBUG(...) fprintf(stderr, "@@ERROR@@" __VA_ARGS__)
+
+#include "llama.h"
+#include "wllama-context.h"
+#include "wllama-fs.h"
+#include "wllama.h"
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+#define WLLAMA_ACTION(name)                \
+  else if (action == #name)                \
+  {                                        \
+    auto res = app.action_##name(req_raw); \
+    res.handler.serialize(output_buffer);  \
+  }
+
+static void llama_log_callback_logTee(ggml_log_level level, const char *text, void *user_data)
+{
+  (void)user_data;
+  const char *lvl = "@@DEBUG";
+  size_t len = strlen(text);
+  if (len == 0 || text[len - 1] != '\n')
+  {
+    // do not print if the line does not terminate with \n
+    return;
+  }
+  if (level == GGML_LOG_LEVEL_ERROR)
+  {
+    lvl = "@@ERROR";
+  }
+  else if (level == GGML_LOG_LEVEL_WARN)
+  {
+    lvl = "@@WARN";
+  }
+  else if (level == GGML_LOG_LEVEL_INFO)
+  {
+    lvl = "@@INFO";
+  }
+  fprintf(stderr, "%s@@%s", lvl, text);
+}
+
+static void printStr(ggml_log_level level, const char *text)
+{
+  std::string str = std::string(text) + "\n";
+  llama_log_callback_logTee(level, str.c_str(), nullptr);
+}
+
+static glue_outbuf output_buffer;
+static wllama_context app;
+
+static std::vector<char> input_buffer;
+// second argument is dummy
+extern "C" const char *wllama_malloc(size_t size, uint32_t)
+{
+  if (input_buffer.size() < size)
+  {
+    input_buffer.resize(size);
+  }
+  return input_buffer.data();
+}
+
+extern "C" const char *wllama_start()
+{
+  try
+  {
+    llama_backend_init();
+    // std::cerr << llama_print_system_info() << "\n";
+    llama_log_set(llama_log_callback_logTee, nullptr);
+    wllama_malloc(1024, 0);
+
+    wllama_fs::make_sure_ready();
+    if (wllama_fs::use_async)
+    {
+      printStr(GGML_LOG_LEVEL_INFO, "Using async file read");
+    }
+
+    return "{\"success\":true}";
+  }
+  catch (std::exception &e)
+  {
+    printStr(GGML_LOG_LEVEL_ERROR, e.what());
+    return "{\"error\":true}";
+  }
+}
+
+extern "C" const char *wllama_action(const char *name, const char *req_raw)
+{
+  try
+  {
+    std::string action(name);
+
+    if (action.empty())
+    {
+      printStr(GGML_LOG_LEVEL_ERROR, "Empty action");
+      abort();
+    }
+
+    WLLAMA_ACTION(load)
+    WLLAMA_ACTION(completion)
+    WLLAMA_ACTION(embedding)
+    WLLAMA_ACTION(rerank)
+    WLLAMA_ACTION(get_result)
+    WLLAMA_ACTION(test_backend_ops)
+
+    else
+    {
+      printStr(GGML_LOG_LEVEL_ERROR, (std::string("Unknown action: ") + name).c_str());
+      abort();
+    }
+
+    // length of response is written inside input_buffer
+    uint32_t *output_len = (uint32_t *)req_raw;
+    output_len[0] = output_buffer.data.size();
+    return output_buffer.data.data();
+  }
+  catch (std::exception &e)
+  {
+    printStr(GGML_LOG_LEVEL_ERROR, e.what());
+    return nullptr;
+  }
+}
+
+extern "C" const char *wllama_exit()
+{
+  try
+  {
+    // app.unload();
+    llama_backend_free();
+    return "{\"success\":true}";
+  }
+  catch (std::exception &e)
+  {
+    printStr(GGML_LOG_LEVEL_ERROR, e.what());
+    return "{\"error\":true}";
+  }
+}
+
+extern "C" const char *wllama_debug()
+{
+  auto get_mem_total = [&]()
+  {
+#ifdef __EMSCRIPTEN__
+    return EM_ASM_INT(return HEAP8.length);
+#else
+    return 0;
+#endif
+  };
+  auto get_mem_free = [&]()
+  {
+#ifdef __EMSCRIPTEN__
+    auto i = mallinfo();
+    size_t total_mem = get_mem_total();
+    size_t dynamic_top = (size_t)sbrk(0);
+    return total_mem - dynamic_top + i.fordblks;
+#else
+    return 0;
+#endif
+  };
+  /*json res = json{
+      {"mem_total_MB", get_mem_total() / 1024 / 1024},
+      {"mem_free_MB", get_mem_free() / 1024 / 1024},
+      {"mem_used_MB", (get_mem_total() - get_mem_free()) / 1024 / 1024},
+  };
+  result = std::string(res.dump());
+  return result.c_str();*/
+  return nullptr;
+}
+
+int main()
+{
+  std::cerr << "Unused\n";
+  return 0;
+}

package/cpp/wllama.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "llama.h"
+#include "common.h"
+
+using llama_tokens = std::vector<llama_token>;

package/esm/cache-manager.d.ts

@@ -0,0 +1,130 @@
+import type { DownloadProgressCallback } from './model-manager';
+import type { StorageBackend } from './storage/index';
+export type DownloadOptions = {
+    /**
+     * Callback function to track download progress
+     */
+    progressCallback?: DownloadProgressCallback;
+    /**
+     * Additional metadata to be stored with the downloaded file
+     */
+    metadataAdditional?: Record<string, any>;
+    /**
+     * Custom headers for the request. Useful for authentication (e.g. Bearer token)
+     */
+    headers?: Record<string, string>;
+    /**
+     * Abort signal for the request
+     */
+    signal?: AbortSignal;
+};
+export declare const POLYFILL_ETAG = "polyfill_for_older_version";
+export interface CacheEntry {
+    /**
+     * Storage key for this file, in the format: `${hashSHA1(fullURL)}_${fileName}`
+     */
+    name: string;
+    /**
+     * Size of file (in bytes)
+     */
+    size: number;
+    /**
+     * Other metadata
+     */
+    metadata: CacheEntryMetadata;
+}
+export interface CacheEntryMetadata {
+    /**
+     * ETag header from remote request
+     * https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
+     */
+    etag: string;
+    /**
+     * Remote file size (in bytes), used for integrity check
+     */
+    originalSize: number;
+    /**
+     * Original URL of the remote model. Unused for now
+     */
+    originalURL: string;
+    /**
+     * URL to mmproj file, if exists
+     */
+    mmprojURL?: string | undefined;
+    /**
+     * Optional SHA256, mostly used by COS backend
+     */
+    sha256?: string | undefined;
+}
+/**
+ * Manages cached model files, backed by a pluggable StorageBackend.
+ *
+ * Defaults to OPFS (Origin Private File System).
+ */
+export declare class CacheManager {
+    private sb;
+    /**
+     * @param backends Array of storage backends to use, in order of preference ; if first is available, use it, otherwise try the next one.
+     */
+    constructor(backends?: StorageBackend[]);
+    /**
+     * Convert a given URL into a storage key.
+     *
+     * Format: `${hashSHA1(fullURL)}_${fileName}`
+     */
+    getNameFromURL(url: string): Promise<string>;
+    /**
+     * @deprecated Use `download()` instead
+     *
+     * Write a new file to cache. This will overwrite existing file.
+     *
+     * @param name The file name returned by `getNameFromURL()` or `list()`
+     */
+    write(name: string, stream: ReadableStream, metadata: CacheEntryMetadata): Promise<void>;
+    download(url: string, options?: DownloadOptions): Promise<void>;
+    /**
+     * Open a file in cache for reading
+     *
+     * @param nameOrURL The file name returned by `getNameFromURL()` or `list()`, or the original URL of the remote file
+     * @returns Blob, or null if file does not exist
+     */
+    open(nameOrURL: string): Promise<Blob | null>;
+    /**
+     * Get the size of a file in stored cache
+     *
+     * NOTE: in case the download is stopped mid-way (i.e. user close browser tab), the file maybe corrupted, size maybe different from `metadata.originalSize`
+     *
+     * @param name The file name returned by `getNameFromURL()` or `list()`
+     * @returns number of bytes, or -1 if file does not exist
+     */
+    getSize(name: string): Promise<number>;
+    /**
+     * Get metadata of a cached file
+     */
+    getMetadata(name: string): Promise<CacheEntryMetadata | null>;
+    /**
+     * List all files currently in cache
+     */
+    list(): Promise<CacheEntry[]>;
+    /**
+     * Clear all files currently in cache
+     */
+    clear(): Promise<void>;
+    /**
+     * Delete a single file in cache
+     *
+     * @param nameOrURL Can be either an URL or a name returned by `getNameFromURL()` or `list()`
+     */
+    delete(nameOrURL: string): Promise<void>;
+    /**
+     * Delete multiple files in cache.
+     *
+     * @param predicate A predicate like `array.filter(item => boolean)`
+     */
+    deleteMany(predicate: (e: CacheEntry) => boolean): Promise<void>;
+    /**
+     * Write the metadata of the file to disk.
+     */
+    writeMetadata(name: string, metadata: CacheEntryMetadata): Promise<void>;
+}
+export default CacheManager;

package/esm/debug.d.ts

@@ -0,0 +1,28 @@
+export declare const Debug: {
+    /**
+     * Resolves a list of wasm function indices to their cleaned symbol names.
+     */
+    decodeFuncIds: (funcIds: number[], isCompatBuild: boolean) => Promise<{
+        funcId: number;
+        name: string;
+    }[]>;
+    /**
+     * Annotates a wasm stack trace string with resolved function names.
+     *
+     * Example input from Chrome:
+     *   at http://localhost:8080/esm/wasm/wllama.wasm:wasm-function[775]:0x74251
+     *   at async blob:http://localhost:8080/53a863cc-7227-45cc-8594-ddbbf5257f20:317:28
+     *
+     * Example input from Firefox:
+     *   @http://localhost:8080/esm/wasm/wllama.wasm:wasm-function[796]:0x7dfe2
+     *       at wModuleInit/WebAssembly.promising/< (9b6a2acd-d909-44e2-b021-d42fb9087cfb:15:32) index.js:1433:45
+     *
+     * Example input from Safari:
+     *   2441@wasm-function[2441]
+     *       at wrapper (d746f19e-4523-4f36-ba06-d0969acc0b05:22:126009)
+     *
+     * Example output:
+     *   wasm-func[775] (server_response::send)
+     */
+    decodeStackTrace: (stack: string, isCompatBuild: boolean) => Promise<string>;
+};

package/esm/glue/glue.d.ts

@@ -0,0 +1,22 @@
+import { type GlueMsg } from './messages';
+/**
+ * Glue is a simple binary protocol for serializing and deserializing messages.
+ * It is inspired by protobuf, but much simpler.
+ *
+ * Interested in extending Glue? Open an issue on GitHub!
+ */
+type GlueType = 'str' | 'int' | 'float' | 'bool' | 'raw' | 'arr_str' | 'arr_int' | 'arr_float' | 'arr_bool' | 'arr_raw' | 'null';
+export interface GlueField {
+    type: GlueType;
+    name: string;
+    isNullable: boolean;
+}
+export interface GlueMessageProto {
+    name: string;
+    structName: string;
+    className: string;
+    fields: GlueField[];
+}
+export declare function glueDeserialize(buf: Uint8Array): GlueMsg;
+export declare function glueSerialize(msg: GlueMsg): Uint8Array;
+export {};

package/esm/glue/messages.d.ts

@@ -0,0 +1,146 @@
+import type { GlueMessageProto } from './glue';
+export declare const GLUE_VERSION = 1;
+export declare const GLUE_MESSAGE_PROTOTYPES: {
+    [name: string]: GlueMessageProto;
+};
+export interface GlueMsgError {
+    _name: "erro_evt";
+    message: string;
+}
+export interface GlueMsgLoadReq {
+    _name: "load_req";
+    model_paths: string[];
+    mmproj_path?: string | undefined;
+    n_ctx_auto: boolean;
+    use_mmap: boolean;
+    use_mlock: boolean;
+    n_gpu_layers: number;
+    n_ctx: number;
+    n_threads: number;
+    model_alias?: string | undefined;
+    log_level?: number | undefined;
+    embeddings?: boolean | undefined;
+    offload_kqv?: boolean | undefined;
+    n_batch?: number | undefined;
+    n_ubatch?: number | undefined;
+    n_parallel?: number | undefined;
+    pooling_type?: string | undefined;
+    rope_scaling_type?: string | undefined;
+    rope_freq_base?: number | undefined;
+    rope_freq_scale?: number | undefined;
+    yarn_ext_factor?: number | undefined;
+    yarn_attn_factor?: number | undefined;
+    yarn_beta_fast?: number | undefined;
+    yarn_beta_slow?: number | undefined;
+    yarn_orig_ctx?: number | undefined;
+    cache_type_k?: string | undefined;
+    cache_type_v?: string | undefined;
+    kv_unified?: boolean | undefined;
+    flash_attn?: boolean | undefined;
+    swa_full?: boolean | undefined;
+    n_ctx_checkpoints?: number | undefined;
+    checkpoint_min_step?: number | undefined;
+    chat_template?: string | undefined;
+    jinja?: boolean | undefined;
+    default_template_kwargs_keys?: string[] | undefined;
+    default_template_kwargs_vals?: string[] | undefined;
+    reasoning?: boolean | undefined;
+    image_min_tokens?: number | undefined;
+    image_max_tokens?: number | undefined;
+    warmup?: boolean | undefined;
+    no_kv_offload?: boolean | undefined;
+    mmproj_offload?: boolean | undefined;
+    cont_batching?: boolean | undefined;
+    n_keep?: number | undefined;
+    ctx_shift?: boolean | undefined;
+    cache_idle_slots?: boolean | undefined;
+    n_cache_reuse?: number | undefined;
+    lora_paths?: string[] | undefined;
+    lora_scales?: number[] | undefined;
+    lora_init_without_apply?: boolean | undefined;
+    spec_draft_model?: string | undefined;
+    spec_draft_ngl?: number | undefined;
+    spec_draft_n_max?: number | undefined;
+    spec_draft_n_min?: number | undefined;
+    spec_draft_p_min?: number | undefined;
+    spec_draft_threads?: number | undefined;
+    spec_draft_threads_batch?: number | undefined;
+    kv_overrides_keys?: string[] | undefined;
+    kv_overrides_vals?: string[] | undefined;
+    reasoning_budget_tokens?: number | undefined;
+    reasoning_budget_message?: string | undefined;
+    reasoning_format?: string | undefined;
+    skip_chat_parsing?: boolean | undefined;
+    prefill_assistant?: boolean | undefined;
+}
+export interface GlueMsgLoadRes {
+    _name: "load_res";
+    success: boolean;
+    n_ctx: number;
+    n_batch: number;
+    n_ubatch: number;
+    n_vocab: number;
+    n_ctx_train: number;
+    n_embd: number;
+    n_layer: number;
+    metadata_key: string[];
+    metadata_val: string[];
+    token_bos: number;
+    token_eos: number;
+    token_eot: number;
+    list_tokens_eog: number[];
+    add_bos_token: boolean;
+    add_eos_token: boolean;
+    has_encoder: boolean;
+    token_decoder_start: number;
+    media_marker: string;
+    has_image_input: boolean;
+    has_audio_input: boolean;
+}
+export interface GlueMsgCompletionReq {
+    _name: "cmpl_req";
+    is_chat: boolean;
+    data_json: string;
+    files: Uint8Array[];
+}
+export interface GlueMsgCompletionRes {
+    _name: "cmpl_res";
+    success: boolean;
+}
+export interface GlueMsgEmbeddingReq {
+    _name: "embd_req";
+    data_json: string;
+    files: Uint8Array[];
+}
+export interface GlueMsgEmbeddingRes {
+    _name: "embd_res";
+    success: boolean;
+}
+export interface GlueMsgRerankReq {
+    _name: "rrnk_req";
+    data_json: string;
+}
+export interface GlueMsgRerankRes {
+    _name: "rrnk_res";
+    success: boolean;
+}
+export interface GlueMsgGetResultReq {
+    _name: "gres_req";
+}
+export interface GlueMsgGetResultRes {
+    _name: "gres_res";
+    success: boolean;
+    has_more: boolean;
+    is_error: boolean;
+    data_json: string;
+}
+export interface GlueMsgTestBackendOpsReq {
+    _name: "tbop_req";
+    args: string[];
+}
+export interface GlueMsgTestBackendOpsRes {
+    _name: "tbop_res";
+    retcode: number;
+    success: boolean;
+}
+export type GlueMsg = GlueMsgError | GlueMsgLoadReq | GlueMsgLoadRes | GlueMsgCompletionReq | GlueMsgCompletionRes | GlueMsgEmbeddingReq | GlueMsgEmbeddingRes | GlueMsgRerankReq | GlueMsgRerankRes | GlueMsgGetResultReq | GlueMsgGetResultRes | GlueMsgTestBackendOpsReq | GlueMsgTestBackendOpsRes;

package/esm/huggingface.d.ts

@@ -0,0 +1,31 @@
+import { type ModelSource } from './model-manager';
+export interface HuggingFaceParams {
+    /**
+     * The repo name, e.g. user/model
+     */
+    repo: string;
+    /**
+     * The file name or path to file in the repo. Only file or quant is needed.
+     */
+    file?: string;
+    /**
+     * The GGUF quantization name, e.g. Q4_K_M, Q8_0, etc. Only file or quant is needed.
+     *
+     * By default, Q4_K_M will be used, then fallback to Q8_0, and finally the non-quantized version if no quantized version is found.
+     */
+    quant?: string;
+    /**
+     * The file name or path to file in the repo for mmproj. Only mmprojFile or mmprojQuant is needed.
+     */
+    mmprojFile?: string;
+    /**
+     * The GGUF quantization name for mmproj, e.g. Q4_K_M, Q8_0, etc. Only mmprojFile or mmprojQuant is needed.
+     */
+    mmprojQuant?: string;
+    /**
+     * The Hugging Face token with permission to access the repo. It can be omitted if the repo is public.
+     */
+    hfToken?: string;
+}
+export declare function getHFModelSource(config: HuggingFaceParams): Promise<ModelSource>;
+export declare function getHFFileSHA256(url: string, headers: Record<string, string>): Promise<string | undefined>;