@fugood/llama.node 0.4.6 → 0.4.7

package/lib/binding.ts

@@ -137,6 +137,10 @@ export type LlamaCompletionToken = {
 
 export type TokenizeResult = {
   tokens: Int32Array
+  has_image: boolean
+  bitmap_hashes: string[]
+  chunk_pos: number[]
+  chunk_pos_images: number[]
 }
 
 export type EmbeddingResult = {
@@ -163,7 +167,7 @@ export interface LlamaContext {
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult>
   stopCompletion(): void
-  tokenize(text: string): Promise<TokenizeResult>
+  tokenize(text: string, image_paths?: string[]): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
   embedding(text: string): Promise<EmbeddingResult>
   saveSession(path: string): Promise<void>

package/lib/index.js

@@ -112,7 +112,7 @@ class LlamaContextWrapper {
     getFormattedChat(messages, template, params) {
         const { messages: chat, has_image, image_paths, } = this._formatImageChat(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
-        let tmpl = this.isLlamaChatSupported() || useJinja ? undefined : 'chatml';
+        let tmpl;
         if (template)
             tmpl = template; // Force replace if provided
         const jsonSchema = getJsonSchema(params === null || params === void 0 ? void 0 : params.response_format);
@@ -144,8 +144,8 @@ class LlamaContextWrapper {
     stopCompletion() {
         return this.ctx.stopCompletion();
     }
-    tokenize(text) {
-        return this.ctx.tokenize(text);
+    tokenize(text, { image_paths } = {}) {
+        return this.ctx.tokenize(text, image_paths);
     }
     detokenize(tokens) {
         return this.ctx.detokenize(tokens);

package/lib/index.ts

@@ -60,6 +60,13 @@ const getJsonSchema = (responseFormat?: CompletionResponseFormat) => {
   return null
 }
 
+export type FormattedChatResult = {
+  type: 'jinja' | 'llama-chat'
+  prompt: string
+  has_image: boolean
+  image_paths?: Array<string>
+}
+
 class LlamaContextWrapper {
   ctx: any
 
@@ -133,7 +140,7 @@ class LlamaContextWrapper {
       parallel_tool_calls?: object
       tool_choice?: string
     },
-  ): object {
+  ): FormattedChatResult {
     const {
       messages: chat,
       has_image,
@@ -141,7 +148,7 @@ class LlamaContextWrapper {
     } = this._formatImageChat(messages)
 
     const useJinja = this.isJinjaSupported() && params?.jinja
-    let tmpl = this.isLlamaChatSupported() || useJinja ? undefined : 'chatml'
+    let tmpl
     if (template) tmpl = template // Force replace if provided
     const jsonSchema = getJsonSchema(params?.response_format)
 
@@ -185,8 +192,8 @@ class LlamaContextWrapper {
     return this.ctx.stopCompletion()
   }
 
-  tokenize(text: string): Promise<TokenizeResult> {
-    return this.ctx.tokenize(text)
+  tokenize(text: string, { image_paths }: { image_paths?: string[] } = {}): Promise<TokenizeResult> {
+    return this.ctx.tokenize(text, image_paths)
   }
 
   detokenize(tokens: number[]): Promise<string> {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.4.6",
+  "version": "0.4.7",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {

package/src/LlamaCompletionWorker.cpp

@@ -1,367 +1,6 @@
 #include "LlamaCompletionWorker.h"
 #include "LlamaContext.h"
 
-// Computes FNV-1a hash of the data
-static std::string fnv_hash(const uint8_t * data, size_t len) {
-  const uint64_t fnv_prime = 0x100000001b3ULL;
-  uint64_t hash = 0xcbf29ce484222325ULL;
-
-  for (size_t i = 0; i < len; ++i) {
-    hash ^= data[i];
-    hash *= fnv_prime;
-  }
-  return std::to_string(hash);
-}
-
-static const std::string base64_chars =
-  "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-  "abcdefghijklmnopqrstuvwxyz"
-  "0123456789+/";
-
-// Base64 decoding function
-static std::vector<uint8_t> base64_decode(const std::string &encoded_string) {
-  std::vector<uint8_t> decoded;
-  int in_len = encoded_string.size();
-  int i = 0;
-  int j = 0;
-  int in_ = 0;
-  unsigned char char_array_4[4], char_array_3[3];
-
-  while (in_len-- && (encoded_string[in_] != '=')) {
-    if (isspace(encoded_string[in_])) {
-      in_++;
-      continue;
-    }
-
-    if (encoded_string[in_] == '=' || base64_chars.find(encoded_string[in_]) == std::string::npos) {
-      break;
-    }
-
-    char_array_4[i++] = encoded_string[in_]; in_++;
-    if (i == 4) {
-      for (i = 0; i < 4; i++) {
-        char_array_4[i] = base64_chars.find(char_array_4[i]);
-      }
-
-      char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
-      char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
-      char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
-
-      for (i = 0; i < 3; i++) {
-        decoded.push_back(char_array_3[i]);
-      }
-      i = 0;
-    }
-  }
-
-  if (i) {
-    for (j = i; j < 4; j++) {
-      char_array_4[j] = 0;
-    }
-
-    for (j = 0; j < 4; j++) {
-      char_array_4[j] = base64_chars.find(char_array_4[j]);
-    }
-
-    char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
-    char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
-    char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
-
-    for (j = 0; j < i - 1; j++) {
-      decoded.push_back(char_array_3[j]);
-    }
-  }
-
-  return decoded;
-}
-
-size_t common_part(const std::vector<llama_token> &a,
-                   const std::vector<llama_token> &b) {
-  size_t i = 0;
-  while (i < a.size() && i < b.size() && a[i] == b[i]) {
-    i++;
-  }
-  return i;
-}
-
-// Process images and add them to the tokenized input
-llama_pos processImage(
-  const mtmd_context* mtmd_ctx,
-  llama_context* ctx,
-  LlamaSessionPtr sess,
-  const std::vector<std::string>& image_paths,
-  const common_params& params,
-  std::vector<llama_token>& text_tokens
-) {
-  if (mtmd_ctx == nullptr) {
-    return false;
-  }
-
-  // Multimodal path
-  std::string full_prompt = params.prompt;
-  // Add image marker if it doesn't already exist
-  if (full_prompt.find("<__image__>") == std::string::npos) {
-    full_prompt += " <__image__>";
-  }
-
-  // Prepare bitmaps array for all images
-  mtmd::bitmaps bitmaps;
-
-  std::vector<std::string> bitmap_hashes;
-
-  // Load all images
-  for (const auto& image_path : image_paths) {
-    fprintf(stdout, "[DEBUG] Loading image: %s\n",
-             image_path.substr(0, 50).c_str()); // Only log part of path for base64
-
-    // Check if it's a base64 image
-    if (image_path.compare(0, 11, "data:image/") == 0) {
-
-      // Parse base64 data
-      std::vector<std::string> parts;
-      size_t comma_pos = image_path.find(',');
-      if (comma_pos == std::string::npos) {
-        bitmaps.entries.clear();
-        return false;
-      }
-
-      std::string header = image_path.substr(0, comma_pos);
-      std::string base64_data = image_path.substr(comma_pos + 1);
-
-      if (header.find("base64") == std::string::npos) {
-        bitmaps.entries.clear();
-        return false;
-      }
-
-      // Decode base64
-      try {
-        // Decode base64 to binary
-        std::vector<uint8_t> image_data = base64_decode(base64_data);
-
-        // Load bitmap from memory buffer using direct initialization
-        mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(image_data.data(), image_data.size()));
-        if (!bmp.ptr) {
-          bitmaps.entries.clear();
-          return false;
-        }
-
-        // Calculate bitmap hash (for KV caching)
-        std::string hash = fnv_hash(bmp.data(), bmp.nx()*bmp.ny()*3);
-        bmp.set_id(hash.c_str());
-        bitmaps.entries.push_back(std::move(bmp));
-        bitmap_hashes.push_back(hash.c_str());
-      } catch (const std::exception& e) {
-        bitmaps.entries.clear();
-        return false;
-      }
-    } else if (image_path.compare(0, 7, "http://") == 0 || image_path.compare(0, 8, "https://") == 0) {
-      // HTTP URLs are not supported yet
-      bitmaps.entries.clear();
-      return false;
-    } else {
-      // Check if file exists
-      FILE* file = fopen(image_path.c_str(), "rb");
-      if (file == nullptr) {
-        bitmaps.entries.clear();
-        return false;
-      }
-
-      // Get file size
-      fseek(file, 0, SEEK_END);
-      long file_size = ftell(file);
-      fseek(file, 0, SEEK_SET);
-      fclose(file);
-
-      // Create bitmap directly
-      mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(image_path.c_str()));
-      if (!bmp.ptr) {
-        bitmaps.entries.clear();
-        return false;
-      }
-
-      // Calculate bitmap hash (for KV caching)
-      std::string hash = fnv_hash(bmp.data(), bmp.nx()*bmp.ny()*3);
-      bmp.set_id(hash.c_str());
-      bitmaps.entries.push_back(std::move(bmp));
-      bitmap_hashes.push_back(hash.c_str());
-    }
-  }
-
-  mtmd_input_chunks* chunks = mtmd_input_chunks_init();
-  if (chunks == nullptr) {
-    bitmaps.entries.clear();
-    return false;
-  }
-
-  // Create input text
-  mtmd_input_text input_text;
-  input_text.text = full_prompt.c_str(); // Use the full prompt with image marker
-  input_text.add_special = true;  // Add BOS token if this is the first message
-  input_text.parse_special = true;       // Parse special tokens like <__image__>
-
-  // Tokenize the text and images
-  fprintf(stdout, "[DEBUG] Tokenizing text and %zu images\n", bitmaps.entries.size());
-  auto bitmaps_c_ptr = bitmaps.c_ptr();
-  
-  // Cast away const for mtmd_tokenize
-  int32_t res = mtmd_tokenize(
-    const_cast<mtmd_context*>(mtmd_ctx), 
-    chunks, 
-    &input_text, 
-    bitmaps_c_ptr.data(), 
-    bitmaps_c_ptr.size()
-  );
-  
-  if (res != 0) {
-    mtmd_input_chunks_free(chunks);
-    bitmaps.entries.clear();
-    return false;
-  }
-
-  // Log chunk information
-  size_t num_chunks = mtmd_input_chunks_size(chunks);
-  fprintf(stdout, "[DEBUG] Tokenization successful: num_chunks=%zu\n", num_chunks);
-
-  // Clear text_tokens before adding new tokens
-  text_tokens.clear();
-
-  // Create a vector to store all tokens (both text and image)
-  std::vector<llama_token> all_tokens;
-
-  // Track the total number of tokens (both text and image)
-  size_t total_token_count = 0;
-
-  // chunk pos
-  std::vector<size_t> chunk_pos;
-  std::vector<size_t> chunk_pos_images;
-  for (size_t i = 0; i < num_chunks; i++) {
-    chunk_pos.push_back(total_token_count);
-
-    const mtmd_input_chunk* chunk = mtmd_input_chunks_get(chunks, i);
-    mtmd_input_chunk_type chunk_type = mtmd_input_chunk_get_type(chunk);
-
-    if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
-      size_t n_tokens;
-      const llama_token* tokens = mtmd_input_chunk_get_tokens_text(chunk, &n_tokens);
-
-      // Add text tokens
-      text_tokens.insert(text_tokens.end(), tokens, tokens + n_tokens);
-      all_tokens.insert(all_tokens.end(), tokens, tokens + n_tokens);
-      total_token_count += n_tokens;
-    } else if (chunk_type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {
-      chunk_pos_images.push_back(total_token_count);
-
-      const mtmd_image_tokens* img_tokens = mtmd_input_chunk_get_tokens_image(chunk);
-      size_t n_tokens = mtmd_image_tokens_get_n_tokens(img_tokens);
-      size_t n_pos = mtmd_image_tokens_get_n_pos(img_tokens);
-
-      for (size_t j = 0; j < n_pos; j++) {
-        all_tokens.push_back(LLAMA_TOKEN_NULL);
-      }
-      total_token_count += n_pos;
-    }
-  }
-
-  llama_pos n_past = common_part(*sess->tokens_ptr(), all_tokens);
-
-  llama_pos new_n_past = n_past;
-
-  // Adjust n_past to position of the text chunk
-  // TODO: Edit the text chunk to remove the tokens before n_past to speed up
-  // need to update the mtmd api
-  auto adjusted_n_past = -1;
-  for (size_t i = 0; i < chunk_pos.size(); i++) {
-    if (n_past < chunk_pos[i]) {
-      break;
-    }
-    bool is_end = i + 1 == chunk_pos.size();
-    if (
-      chunk_pos[i] < n_past &&
-      (!is_end && chunk_pos[i + 1] > n_past)
-      // is_end & n_past < total_token_count:
-      // don't need to adjust and it will skip eval_chunk_single, let nextToken() to finish the job
-    ) {
-      adjusted_n_past = chunk_pos[i];
-    }
-  }
-  if (adjusted_n_past != -1) {
-    n_past = adjusted_n_past;
-    new_n_past = n_past;
-    fprintf(stdout, "[DEBUG] Adjusted n_past to %d\n", n_past);
-  }
-
-  // Compare bitmap hashes, if they are not the same, backtrack n_past to the position of the first mismatch
-  auto mtmd_bitmap_past_hashes = sess->mtmd_bitmap_past_hashes_ptr();
-  if (mtmd_bitmap_past_hashes->size() > 0) {
-    for (size_t i = 0; i < bitmap_hashes.size(); i++) {
-      auto pos = chunk_pos_images[i];
-      if (n_past < pos) {
-        break;
-      }
-      if (i >= mtmd_bitmap_past_hashes->size()) {
-        break;
-      }
-      if (bitmap_hashes[i] != (*mtmd_bitmap_past_hashes)[i]) {
-        n_past = chunk_pos_images[i];
-        new_n_past = n_past;
-        break;
-      }
-    }
-  }
-
-  // Clear all KV cache entries after position n_past
-  llama_kv_self_seq_rm(ctx, 0, n_past, -1);
-
-  for (size_t i = 0; i < chunk_pos.size(); i++) {
-    fprintf(stdout, "[DEBUG] Evaluating chunk %zu: n_past=%d, chunk_pos=%zu\n", i, n_past, chunk_pos[i]);
-
-    // Process chunk only if it's after the current n_past
-    if (chunk_pos[i] >= new_n_past) {
-      bool chunk_logits_last = (i == num_chunks - 1);
-      auto chunk = mtmd_input_chunks_get(chunks, i);
-
-      // Cast away const for mtmd_helper_eval_chunk_single
-      int32_t res = mtmd_helper_eval_chunk_single(
-        const_cast<mtmd_context*>(mtmd_ctx),
-        ctx,
-        chunk,
-        n_past,
-        0,
-        params.n_batch, // batch size
-        chunk_logits_last,
-        &new_n_past
-      );
-      
-      if (res != 0) {
-        mtmd_input_chunks_free(chunks);
-        bitmaps.entries.clear();
-        return false;
-      }
-      n_past = new_n_past;
-    }
-  }
-
-  if (n_past == total_token_count && n_past > 0 && all_tokens[n_past - 1] != LLAMA_TOKEN_NULL) {
-    // we have to evaluate at least 1 token to generate logits.
-    n_past--;
-  }
-
-  // Update sampling context to process token sequences
-  for (auto & token : all_tokens) {
-    if (token == LLAMA_TOKEN_NULL) {
-      continue;
-    }
-  }
-  // Set the tokens
-  sess->set_tokens(std::move(all_tokens));
-
-  sess->set_mtmd_bitmap_past_hashes(bitmap_hashes);
-
-  // Clean up image resources
-  mtmd_input_chunks_free(chunks);
-  bitmaps.entries.clear();
-  return n_past;
-}
 
 size_t findStoppingStrings(const std::string &text,
                            const size_t last_token_size,
@@ -425,22 +64,25 @@ void LlamaCompletionWorker::Execute() {
   LlamaCppSampling sampling{common_sampler_init(model, _params.sampling),
                             common_sampler_free};
 
-  std::vector<llama_token> prompt_tokens;
-
   // Process images if any are provided
   if (!_image_paths.empty()) {
     const auto* mtmd_ctx = _sess->get_mtmd_ctx();
     
     if (mtmd_ctx != nullptr) {
       // Process the images and get the tokens
-      n_cur = processImage(
-        mtmd_ctx,
-        ctx,
-        _sess,
-        _image_paths,
-        _params,
-        prompt_tokens
-      );
+      try {
+        n_cur = process_image_prompt(
+          ctx,
+          mtmd_ctx,
+          _sess,
+          _params,
+          _image_paths
+        );
+      } catch (const std::exception& e) {
+        SetError(e.what());
+        _sess->get_mutex().unlock();
+        return;
+      }
       
       if (n_cur <= 0) {
         SetError("Failed to process images");
@@ -456,7 +98,6 @@ void LlamaCompletionWorker::Execute() {
         --n_cur;
       }
       n_input -= n_cur;
-      llama_kv_self_seq_rm(ctx, 0, n_cur, -1);
     } else {
       SetError("Multimodal context not initialized");
       _sess->get_mutex().unlock();
@@ -464,11 +105,11 @@ void LlamaCompletionWorker::Execute() {
     }
   } else {
     // Text-only path
-    prompt_tokens = ::common_tokenize(ctx, _params.prompt, add_bos);
+    std::vector<llama_token> prompt_tokens = ::common_tokenize(ctx, _params.prompt, add_bos);
     n_input = prompt_tokens.size();
     
     if (_sess->tokens_ptr()->size() > 0) {
-      n_cur = common_part(*(_sess->tokens_ptr()), prompt_tokens);
+      n_cur = common_tokens_part(*(_sess->tokens_ptr()), prompt_tokens);
       if (n_cur == n_input) {
         --n_cur;
       }

package/src/LlamaCompletionWorker.h

@@ -4,8 +4,6 @@
 #include <atomic>
 #include <functional>
 #include <napi.h>
-#include "tools/mtmd/mtmd.h"
-#include "tools/mtmd/clip.h"
 
 struct CompletionResult {
   std::string text = "";

package/src/LlamaContext.cpp

@@ -27,80 +27,6 @@ static std::string format_string(const std::string& format, Args ... args) {
     return std::string(buf.get(), buf.get() + size - 1); // -1 to exclude null terminator
 }
 
-// Computes FNV-1a hash of the data
-static std::string fnv_hash(const uint8_t* data, size_t len) {
-  const uint64_t fnv_prime = 0x100000001b3ULL;
-  uint64_t hash = 0xcbf29ce484222325ULL;
-
-  for (size_t i = 0; i < len; ++i) {
-    hash ^= data[i];
-    hash *= fnv_prime;
-  }
-  return std::to_string(hash);
-}
-
-static const std::string base64_chars =
-  "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-  "abcdefghijklmnopqrstuvwxyz"
-  "0123456789+/";
-
-// Base64 decoding function
-static std::vector<uint8_t> base64_decode(const std::string &encoded_string) {
-  std::vector<uint8_t> decoded;
-  int in_len = encoded_string.size();
-  int i = 0;
-  int j = 0;
-  int in_ = 0;
-  unsigned char char_array_4[4], char_array_3[3];
-
-  while (in_len-- && (encoded_string[in_] != '=')) {
-    if (isspace(encoded_string[in_])) {
-      in_++;
-      continue;
-    }
-
-    if (encoded_string[in_] == '=' || base64_chars.find(encoded_string[in_]) == std::string::npos) {
-      break;
-    }
-
-    char_array_4[i++] = encoded_string[in_]; in_++;
-    if (i == 4) {
-      for (i = 0; i < 4; i++) {
-        char_array_4[i] = base64_chars.find(char_array_4[i]);
-      }
-
-      char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
-      char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
-      char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
-
-      for (i = 0; i < 3; i++) {
-        decoded.push_back(char_array_3[i]);
-      }
-      i = 0;
-    }
-  }
-
-  if (i) {
-    for (j = i; j < 4; j++) {
-      char_array_4[j] = 0;
-    }
-
-    for (j = 0; j < 4; j++) {
-      char_array_4[j] = base64_chars.find(char_array_4[j]);
-    }
-
-    char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
-    char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
-    char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
-
-    for (j = 0; j < i - 1; j++) {
-      decoded.push_back(char_array_3[j]);
-    }
-  }
-
-  return decoded;
-}
-
 using json = nlohmann::ordered_json;
 
 // loadModelInfo(path: string): object
@@ -153,18 +79,6 @@ Napi::Value LlamaContext::ModelInfo(const Napi::CallbackInfo& info) {
   return metadata;
 }
 
-std::vector<common_chat_msg> get_messages(Napi::Array messages) {
-  std::vector<common_chat_msg> chat;
-  for (size_t i = 0; i < messages.Length(); i++) {
-    auto message = messages.Get(i).As<Napi::Object>();
-    chat.push_back({
-      get_option<std::string>(message, "role", ""),
-      get_option<std::string>(message, "content", ""),
-    });
-  }
-  return std::move(chat);
-}
-
 void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
   Napi::Function func = DefineClass(
       env, "LlamaContext",
@@ -919,7 +833,14 @@ Napi::Value LlamaContext::Tokenize(const Napi::CallbackInfo &info) {
         .ThrowAsJavaScriptException();
   }
   auto text = info[0].ToString().Utf8Value();
-  auto *worker = new TokenizeWorker(info, _sess, text);
+  std::vector<std::string> image_paths;
+  if (info.Length() >= 2 && info[1].IsArray()) {
+    auto image_paths_array = info[1].As<Napi::Array>();
+    for (size_t i = 0; i < image_paths_array.Length(); i++) {
+      image_paths.push_back(image_paths_array.Get(i).ToString().Utf8Value());
+    }
+  }
+  auto *worker = new TokenizeWorker(info, _sess, text, image_paths);
   worker->Queue();
   return worker->Promise();
 }

package/src/TokenizeWorker.cpp

@@ -2,12 +2,18 @@
 #include "LlamaContext.h"
 
 TokenizeWorker::TokenizeWorker(const Napi::CallbackInfo &info,
-                               LlamaSessionPtr &sess, std::string text)
-    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text) {}
+                               LlamaSessionPtr &sess, std::string text, std::vector<std::string> image_paths)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text), _image_paths(image_paths) {}
 
 void TokenizeWorker::Execute() {
-  const auto tokens = ::common_tokenize(_sess->context(), _text, false);
-  _result.tokens = std::move(tokens);
+  auto mtmd_ctx = _sess->get_mtmd_ctx();
+  if (!_image_paths.empty()) {
+    _result = tokenizeWithImages(mtmd_ctx, _text, _image_paths);
+  } else {
+    const auto tokens = common_tokenize(_sess->context(), _text, false);
+    _result.tokens = tokens;
+    _result.has_image = false;
+  }
 }
 
 void TokenizeWorker::OnOK() {
@@ -18,6 +24,25 @@ void TokenizeWorker::OnOK() {
   memcpy(tokens.Data(), _result.tokens.data(),
          _result.tokens.size() * sizeof(llama_token));
   result.Set("tokens", tokens);
+  if (_result.has_image) {
+    result.Set("has_image", _result.has_image);
+
+    auto bitmap_hashes = Napi::Array::New(Napi::AsyncWorker::Env(), _result.bitmap_hashes.size());
+    for (size_t i = 0; i < _result.bitmap_hashes.size(); i++) {
+      bitmap_hashes.Set(i, _result.bitmap_hashes[i]);
+    }
+    result.Set("bitmap_hashes", bitmap_hashes);
+    auto chunk_pos = Napi::Array::New(Napi::AsyncWorker::Env(), _result.chunk_pos.size());
+    for (size_t i = 0; i < _result.chunk_pos.size(); i++) {
+      chunk_pos.Set(i, _result.chunk_pos[i]);
+    }
+    result.Set("chunk_pos", chunk_pos);
+    auto chunk_pos_images = Napi::Array::New(Napi::AsyncWorker::Env(), _result.chunk_pos_images.size());
+    for (size_t i = 0; i < _result.chunk_pos_images.size(); i++) {
+      chunk_pos_images.Set(i, _result.chunk_pos_images[i]);
+    }
+    result.Set("chunk_pos_images", chunk_pos_images);
+  }
   Napi::Promise::Deferred::Resolve(result);
 }
 

package/src/TokenizeWorker.h

@@ -1,15 +1,11 @@
 #include "common.hpp"
 #include <vector>
 
-struct TokenizeResult {
-  std::vector<llama_token> tokens;
-};
-
 class TokenizeWorker : public Napi::AsyncWorker,
                        public Napi::Promise::Deferred {
 public:
   TokenizeWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
-                 std::string text);
+                 std::string text, std::vector<std::string> image_paths);
 
 protected:
   void Execute();
@@ -19,5 +15,6 @@ protected:
 private:
   LlamaSessionPtr _sess;
   std::string _text;
+  std::vector<std::string> _image_paths;
   TokenizeResult _result;
 };

package/src/common.hpp

@@ -2,6 +2,8 @@
 
 #include "common/common.h"
 #include "common/sampling.h"
+#include "tools/mtmd/mtmd.h"
+#include "tools/mtmd/clip.h"
 #include "chat.h"
 #include "llama.h"
 #include "tools/mtmd/mtmd.h"
@@ -120,3 +122,386 @@ private:
 };
 
 typedef std::shared_ptr<LlamaSession> LlamaSessionPtr;
+
+static size_t common_tokens_part(const std::vector<llama_token> &a,
+                   const std::vector<llama_token> &b) {
+  size_t i = 0;
+  while (i < a.size() && i < b.size() && a[i] == b[i]) {
+    i++;
+  }
+  return i;
+}
+
+// Computes FNV-1a hash of the data
+static std::string fnv_hash(const uint8_t * data, size_t len) {
+  const uint64_t fnv_prime = 0x100000001b3ULL;
+  uint64_t hash = 0xcbf29ce484222325ULL;
+
+  for (size_t i = 0; i < len; ++i) {
+    hash ^= data[i];
+    hash *= fnv_prime;
+  }
+  return std::to_string(hash);
+}
+
+static const std::string base64_chars =
+  "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+  "abcdefghijklmnopqrstuvwxyz"
+  "0123456789+/";
+
+// Base64 decoding function
+static std::vector<uint8_t> base64_decode(const std::string &encoded_string) {
+  std::vector<uint8_t> decoded;
+  int in_len = encoded_string.size();
+  int i = 0;
+  int j = 0;
+  int in_ = 0;
+  unsigned char char_array_4[4], char_array_3[3];
+
+  while (in_len-- && (encoded_string[in_] != '=')) {
+    if (isspace(encoded_string[in_])) {
+      in_++;
+      continue;
+    }
+
+    if (encoded_string[in_] == '=' || base64_chars.find(encoded_string[in_]) == std::string::npos) {
+      break;
+    }
+
+    char_array_4[i++] = encoded_string[in_]; in_++;
+    if (i == 4) {
+      for (i = 0; i < 4; i++) {
+        char_array_4[i] = base64_chars.find(char_array_4[i]);
+      }
+
+      char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
+      char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
+      char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
+
+      for (i = 0; i < 3; i++) {
+        decoded.push_back(char_array_3[i]);
+      }
+      i = 0;
+    }
+  }
+
+  if (i) {
+    for (j = i; j < 4; j++) {
+      char_array_4[j] = 0;
+    }
+
+    for (j = 0; j < 4; j++) {
+      char_array_4[j] = base64_chars.find(char_array_4[j]);
+    }
+
+    char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
+    char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
+    char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
+
+    for (j = 0; j < i - 1; j++) {
+      decoded.push_back(char_array_3[j]);
+    }
+  }
+
+  return decoded;
+}
+
+struct TokenizeResult {
+  std::vector<llama_token> tokens;
+
+  bool has_image = false;
+  std::vector<std::string> bitmap_hashes;
+  std::vector<size_t> chunk_pos; // both text and image
+  std::vector<size_t> chunk_pos_images; // image only
+  mtmd_input_chunks* chunks = nullptr;
+};
+
+static TokenizeResult tokenizeWithImages(
+  const mtmd_context* mtmd_ctx,
+  const std::string &prompt,
+  const std::vector<std::string> &image_paths
+) {
+  if (mtmd_ctx == nullptr) {
+    throw std::runtime_error("Multimodal context is not initialized");
+  }
+
+  TokenizeResult result;
+  result.has_image = !image_paths.empty();
+
+  mtmd::bitmaps bitmaps;
+
+  // Load all images
+  for (const auto& image_path : image_paths) {
+    fprintf(stdout, "[DEBUG] Loading image: %s\n",
+             image_path.substr(0, 50).c_str()); // Only log part of path for base64
+
+    // Check if it's a base64 image
+    if (image_path.compare(0, 11, "data:image/") == 0) {
+
+      // Parse base64 data
+      std::vector<std::string> parts;
+      size_t comma_pos = image_path.find(',');
+      if (comma_pos == std::string::npos) {
+        result.bitmap_hashes.clear();
+        throw std::runtime_error("Invalid base64 image");
+      }
+
+      std::string header = image_path.substr(0, comma_pos);
+      std::string base64_data = image_path.substr(comma_pos + 1);
+
+      if (header.find("base64") == std::string::npos) {
+        result.bitmap_hashes.clear();
+        throw std::runtime_error("Invalid base64 image");
+      }
+
+      // Decode base64
+      try {
+        // Decode base64 to binary
+        std::vector<uint8_t> image_data = base64_decode(base64_data);
+
+        // Load bitmap from memory buffer using direct initialization
+        mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(image_data.data(), image_data.size()));
+        if (!bmp.ptr) {
+          bitmaps.entries.clear();
+          throw std::runtime_error("Failed to decode base64 image");
+        }
+
+        // Calculate bitmap hash (for KV caching)
+        std::string hash = fnv_hash(bmp.data(), bmp.nx()*bmp.ny()*3);
+        bmp.set_id(hash.c_str());
+        bitmaps.entries.push_back(std::move(bmp));
+        result.bitmap_hashes.push_back(hash.c_str());
+      } catch (const std::exception& e) {
+        bitmaps.entries.clear();
+        throw std::runtime_error("Failed to decode base64 image");
+      }
+    } else if (image_path.compare(0, 7, "http://") == 0 || image_path.compare(0, 8, "https://") == 0) {
+      // HTTP URLs are not supported yet
+      bitmaps.entries.clear();
+      throw std::runtime_error("HTTP URLs are not supported yet");
+    } else {
+      // Check if file exists
+      FILE* file = fopen(image_path.c_str(), "rb");
+      if (file == nullptr) {
+        bitmaps.entries.clear();
+        throw std::runtime_error("Failed to open image file");
+      }
+
+      // Get file size
+      fseek(file, 0, SEEK_END);
+      long file_size = ftell(file);
+      fseek(file, 0, SEEK_SET);
+      fclose(file);
+
+      // Create bitmap directly
+      mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(image_path.c_str()));
+      if (!bmp.ptr) {
+        bitmaps.entries.clear();
+        throw std::runtime_error("Failed to create bitmap from image file");
+      }
+
+      // Calculate bitmap hash (for KV caching)
+      std::string hash = fnv_hash(bmp.data(), bmp.nx()*bmp.ny()*3);
+      bmp.set_id(hash.c_str());
+      bitmaps.entries.push_back(std::move(bmp));
+      result.bitmap_hashes.push_back(hash.c_str());
+    }
+  }
+
+  result.chunks = mtmd_input_chunks_init();
+  if (result.chunks == nullptr) {
+    bitmaps.entries.clear();
+    throw std::runtime_error("Failed to initialize input chunks");
+  }
+  
+  // Create input text
+  mtmd_input_text input_text;
+  input_text.text = prompt.c_str(); // Use the full prompt with image marker
+  input_text.add_special = true;  // Add BOS token if this is the first message
+  input_text.parse_special = true;       // Parse special tokens like <__image__>
+
+  // Tokenize the text and images
+  fprintf(stdout, "[DEBUG] Tokenizing text and %zu images\n", bitmaps.entries.size());
+  auto bitmaps_c_ptr = bitmaps.c_ptr();
+  
+  // Cast away const for mtmd_tokenize
+  int32_t res = mtmd_tokenize(
+    const_cast<mtmd_context*>(mtmd_ctx), 
+    result.chunks, 
+    &input_text, 
+    bitmaps_c_ptr.data(), 
+    bitmaps_c_ptr.size()
+  );
+  
+  if (res != 0) {
+    mtmd_input_chunks_free(result.chunks);
+    bitmaps.entries.clear();
+    throw std::runtime_error("Failed to tokenize text and images");
+  }
+
+  // Log chunk information
+  size_t num_chunks = mtmd_input_chunks_size(result.chunks);
+  fprintf(stdout, "[DEBUG] Tokenization successful: num_chunks=%zu\n", num_chunks);
+
+  // Track the total number of tokens (both text and image)
+  size_t total_token_count = 0;
+
+  // chunk pos
+  for (size_t i = 0; i < num_chunks; i++) {
+    result.chunk_pos.push_back(total_token_count);
+
+    const mtmd_input_chunk* chunk = mtmd_input_chunks_get(result.chunks, i);
+    mtmd_input_chunk_type chunk_type = mtmd_input_chunk_get_type(chunk);
+
+    if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
+      size_t n_tokens;
+      const llama_token* tokens = mtmd_input_chunk_get_tokens_text(chunk, &n_tokens);
+
+      result.tokens.insert(result.tokens.end(), tokens, tokens + n_tokens);
+      total_token_count += n_tokens;
+    } else if (chunk_type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {
+      result.chunk_pos_images.push_back(total_token_count);
+
+      const mtmd_image_tokens* img_tokens = mtmd_input_chunk_get_tokens_image(chunk);
+      size_t n_tokens = mtmd_image_tokens_get_n_tokens(img_tokens);
+      size_t n_pos = mtmd_image_tokens_get_n_pos(img_tokens);
+
+      for (size_t j = 0; j < n_pos; j++) {
+        result.tokens.push_back(LLAMA_TOKEN_NULL);
+      }
+      total_token_count += n_pos;
+    }
+  }
+
+  bitmaps.entries.clear();
+  
+  return result;
+}
+
+// Process images and add them to the tokenized input
+static llama_pos process_image_prompt(
+  llama_context* ctx,
+  const mtmd_context* mtmd_ctx,
+  LlamaSessionPtr sess,
+  const common_params& params,  
+  const std::vector<std::string>& image_paths
+) {
+  if (mtmd_ctx == nullptr) {
+    throw std::runtime_error("Multimodal context is not initialized");
+  }
+
+  // Multimodal path
+  std::string full_prompt = params.prompt;
+  // Add image marker if it doesn't already exist
+  if (full_prompt.find("<__image__>") == std::string::npos) {
+    full_prompt += " <__image__>";
+  }
+
+  auto result = tokenizeWithImages(mtmd_ctx, full_prompt, image_paths);
+
+  auto all_tokens = result.tokens;
+  auto chunks = result.chunks;
+  auto chunk_pos = result.chunk_pos;
+  auto chunk_pos_images = result.chunk_pos_images;
+  auto bitmap_hashes = result.bitmap_hashes;
+
+  llama_pos n_past = common_tokens_part(*sess->tokens_ptr(), all_tokens);
+
+  llama_pos new_n_past = n_past;
+
+  // Adjust n_past to position of the text chunk
+  // TODO: Edit the text chunk to remove the tokens before n_past to speed up
+  // need to update the mtmd api
+  auto adjusted_n_past = -1;
+  for (size_t i = 0; i < chunk_pos.size(); i++) {
+    if (n_past < chunk_pos[i]) {
+      break;
+    }
+    bool is_end = i + 1 == chunk_pos.size();
+    if (
+      chunk_pos[i] < n_past &&
+      (!is_end && chunk_pos[i + 1] > n_past)
+      // is_end & n_past < total_token_count:
+      // don't need to adjust and it will skip eval_chunk_single, let nextToken() to finish the job
+    ) {
+      adjusted_n_past = chunk_pos[i];
+    }
+  }
+  if (adjusted_n_past != -1) {
+    n_past = adjusted_n_past;
+    new_n_past = n_past;
+    fprintf(stdout, "[DEBUG] Adjusted n_past to %d\n", n_past);
+  }
+
+  // Compare bitmap hashes, if they are not the same, backtrack n_past to the position of the first mismatch
+  auto mtmd_bitmap_past_hashes = sess->mtmd_bitmap_past_hashes_ptr();
+  if (mtmd_bitmap_past_hashes->size() > 0) {
+    for (size_t i = 0; i < bitmap_hashes.size(); i++) {
+      auto pos = chunk_pos_images[i];
+      if (n_past < pos) {
+        break;
+      }
+      if (i >= mtmd_bitmap_past_hashes->size()) {
+        break;
+      }
+      if (bitmap_hashes[i] != (*mtmd_bitmap_past_hashes)[i]) {
+        n_past = chunk_pos_images[i];
+        new_n_past = n_past;
+        break;
+      }
+    }
+  }
+
+  // Clear all KV cache entries after position n_past
+  llama_kv_self_seq_rm(ctx, 0, n_past, -1);
+
+  size_t num_chunks = mtmd_input_chunks_size(chunks);
+
+  for (size_t i = 0; i < chunk_pos.size(); i++) {
+    fprintf(stdout, "[DEBUG] Evaluating chunk %zu: n_past=%d, chunk_pos=%zu\n", i, n_past, chunk_pos[i]);
+
+    // Process chunk only if it's after the current n_past
+    if (chunk_pos[i] >= new_n_past) {
+      bool chunk_logits_last = (i == num_chunks - 1);
+      auto chunk = mtmd_input_chunks_get(chunks, i);
+
+      // Cast away const for mtmd_helper_eval_chunk_single
+      int32_t res = mtmd_helper_eval_chunk_single(
+        const_cast<mtmd_context*>(mtmd_ctx),
+        ctx,
+        chunk,
+        n_past,
+        0,
+        params.n_batch, // batch size
+        chunk_logits_last,
+        &new_n_past
+      );
+      
+      if (res != 0) {
+        mtmd_input_chunks_free(chunks);
+        throw std::runtime_error("Failed to process chunk");
+      }
+      n_past = new_n_past;
+    }
+  }
+
+  if (n_past == all_tokens.size() && n_past > 0 && all_tokens[n_past - 1] != LLAMA_TOKEN_NULL) {
+    // we have to evaluate at least 1 token to generate logits.
+    n_past--;
+  }
+
+  // Update sampling context to process token sequences
+  for (auto & token : all_tokens) {
+    if (token == LLAMA_TOKEN_NULL) {
+      continue;
+    }
+  }
+  // Set the tokens
+  sess->set_tokens(std::move(all_tokens));
+
+  sess->set_mtmd_bitmap_past_hashes(bitmap_hashes);
+
+  // Clean up image resources
+  mtmd_input_chunks_free(chunks);
+  return n_past;
+}