@fugood/llama.node 0.3.16 → 0.4.0

package/CMakeLists.txt

@@ -69,12 +69,17 @@ endif()
 
 set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build common")
 
+set(LLAMA_CURL OFF CACHE BOOL "Build curl")
+
 set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libraries")
 add_subdirectory("src/llama.cpp")
+add_subdirectory("src/llama.cpp/tools/mtmd")
 
 include_directories(
   ${CMAKE_JS_INC}
   "src/llama.cpp"
+  "src/llama.cpp/src"
+  "src/tools/mtmd"
 )
 
 file(
@@ -101,7 +106,7 @@ file(
 
 add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC})
 set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
-target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB} llama ggml common)
+target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB} llama ggml common mtmd ${CMAKE_THREAD_LIBS_INIT})
 
 add_custom_target(copy_assets ALL DEPENDS ${PROJECT_NAME})
 

package/lib/binding.ts

@@ -1,8 +1,17 @@
 import * as path from 'path'
 
+
+export type MessagePart = {
+  type: string,
+  text?: string,
+  image_url?: {
+    url?: string
+  }
+}
+
 export type ChatMessage = {
   role: string
-  content: string
+  content?: string | MessagePart[]
 }
 
 export type LlamaModelOptions = {
@@ -36,6 +45,10 @@ export type LlamaModelOptions = {
     | 'iq4_nl'
     | 'q5_0'
     | 'q5_1'
+  /**
+   * Enable context shifting to handle prompts larger than context size
+   */
+  ctx_shift?: boolean
   use_mlock?: boolean
   use_mmap?: boolean
   vocab_only?: boolean
@@ -89,6 +102,13 @@ export type LlamaCompletionOptions = {
   grammar_lazy?: boolean
   grammar_triggers?: { type: number; word: string; at_start: boolean }[]
   preserved_tokens?: string[]
+  /**
+   * Path(s) to image file(s) to process before generating text.
+   * When provided, the image(s) will be processed and added to the context.
+   * Requires multimodal support to be enabled via initMultimodal.
+   * Supports both file paths and base64 data URLs.
+   */
+  image_paths?: string | string[]
 }
 
 export type LlamaCompletionResult = {
@@ -96,6 +116,7 @@ export type LlamaCompletionResult = {
   tokens_predicted: number
   tokens_evaluated: number
   truncated: boolean
+  context_full: boolean
   timings: {
     prompt_n: number
     prompt_ms: number
@@ -149,9 +170,30 @@ export interface LlamaContext {
   applyLoraAdapters(adapters: { path: string; scaled: number }[]): void
   removeLoraAdapters(adapters: { path: string }[]): void
   getLoadedLoraAdapters(): { path: string; scaled: number }[]
+  /**
+   * Initialize multimodal support with a mmproj file
+   * @param mmproj_path Path to the multimodal projector file
+   * @returns Promise resolving to true if initialization was successful
+   */
+  initMultimodal(options: { path: string; use_gpu?: boolean }): Promise<boolean>
+
+  /**
+   * Check if multimodal support is enabled
+   * @returns Promise resolving to true if multimodal is enabled
+   */
+  isMultimodalEnabled(): Promise<boolean>
+
+  /**
+   * Release multimodal support
+   */
+  releaseMultimodal(): Promise<void>
+
   // static
   loadModelInfo(path: string, skip: string[]): Promise<Object>
-  toggleNativeLog(enable: boolean, callback: (level: string, text: string) => void): void
+  toggleNativeLog(
+    enable: boolean,
+    callback: (level: string, text: string) => void,
+  ): void
 }
 
 export interface Module {

package/lib/index.js

@@ -51,12 +51,143 @@ function addNativeLogListener(listener) {
         },
     };
 }
+const getJsonSchema = (responseFormat) => {
+    var _a;
+    if ((responseFormat === null || responseFormat === void 0 ? void 0 : responseFormat.type) === 'json_schema') {
+        return (_a = responseFormat.json_schema) === null || _a === void 0 ? void 0 : _a.schema;
+    }
+    if ((responseFormat === null || responseFormat === void 0 ? void 0 : responseFormat.type) === 'json_object') {
+        return responseFormat.schema || {};
+    }
+    return null;
+};
+class LlamaContextWrapper {
+    constructor(nativeCtx) {
+        this.ctx = nativeCtx;
+    }
+    getSystemInfo() {
+        return this.ctx.getSystemInfo();
+    }
+    getModelInfo() {
+        return this.ctx.getModelInfo();
+    }
+    isJinjaSupported() {
+        const { minja } = this.ctx.getModelInfo().chatTemplates;
+        return !!(minja === null || minja === void 0 ? void 0 : minja.toolUse) || !!(minja === null || minja === void 0 ? void 0 : minja.default);
+    }
+    isLlamaChatSupported() {
+        return !!this.ctx.getModelInfo().chatTemplates.llamaChat;
+    }
+    _formatImageChat(messages) {
+        if (!messages)
+            return {
+                messages,
+                has_image: false,
+            };
+        const imagePaths = [];
+        return {
+            messages: messages.map((msg) => {
+                if (Array.isArray(msg.content)) {
+                    const content = msg.content.map((part) => {
+                        var _a;
+                        // Handle multimodal content
+                        if (part.type === 'image_url') {
+                            let path = ((_a = part.image_url) === null || _a === void 0 ? void 0 : _a.url) || '';
+                            imagePaths.push(path);
+                            return {
+                                type: 'text',
+                                text: '<__image__>',
+                            };
+                        }
+                        return part;
+                    });
+                    return Object.assign(Object.assign({}, msg), { content });
+                }
+                return msg;
+            }),
+            has_image: imagePaths.length > 0,
+            image_paths: imagePaths,
+        };
+    }
+    getFormattedChat(messages, template, params) {
+        const { messages: chat, has_image, image_paths, } = this._formatImageChat(messages);
+        const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
+        let tmpl = this.isLlamaChatSupported() || useJinja ? undefined : 'chatml';
+        if (template)
+            tmpl = template; // Force replace if provided
+        const jsonSchema = getJsonSchema(params === null || params === void 0 ? void 0 : params.response_format);
+        const result = this.ctx.getFormattedChat(chat, tmpl, {
+            jinja: useJinja,
+            json_schema: jsonSchema,
+            tools: params === null || params === void 0 ? void 0 : params.tools,
+            parallel_tool_calls: params === null || params === void 0 ? void 0 : params.parallel_tool_calls,
+            tool_choice: params === null || params === void 0 ? void 0 : params.tool_choice,
+        });
+        if (!useJinja) {
+            return {
+                type: 'llama-chat',
+                prompt: result,
+                has_image,
+                image_paths,
+            };
+        }
+        const jinjaResult = result;
+        jinjaResult.type = 'jinja';
+        jinjaResult.has_image = has_image;
+        jinjaResult.image_paths = image_paths;
+        return jinjaResult;
+    }
+    completion(options, callback) {
+        const { messages, image_paths = options.image_paths } = this._formatImageChat(options.messages);
+        return this.ctx.completion(Object.assign(Object.assign({}, options), { messages, image_paths: options.image_paths || image_paths }), callback || (() => { }));
+    }
+    stopCompletion() {
+        return this.ctx.stopCompletion();
+    }
+    tokenize(text) {
+        return this.ctx.tokenize(text);
+    }
+    detokenize(tokens) {
+        return this.ctx.detokenize(tokens);
+    }
+    embedding(text) {
+        return this.ctx.embedding(text);
+    }
+    saveSession(path) {
+        return this.ctx.saveSession(path);
+    }
+    loadSession(path) {
+        return this.ctx.loadSession(path);
+    }
+    release() {
+        return this.ctx.release();
+    }
+    applyLoraAdapters(adapters) {
+        return this.ctx.applyLoraAdapters(adapters);
+    }
+    removeLoraAdapters(adapters) {
+        return this.ctx.removeLoraAdapters(adapters);
+    }
+    getLoadedLoraAdapters() {
+        return this.ctx.getLoadedLoraAdapters();
+    }
+    initMultimodal(options) {
+        return this.ctx.initMultimodal(options);
+    }
+    isMultimodalEnabled() {
+        return this.ctx.isMultimodalEnabled();
+    }
+    releaseMultimodal() {
+        return this.ctx.releaseMultimodal();
+    }
+}
 const loadModel = (options) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;
     const variant = (_a = options.lib_variant) !== null && _a !== void 0 ? _a : 'default';
     (_b = mods[variant]) !== null && _b !== void 0 ? _b : (mods[variant] = yield (0, binding_1.loadModule)(options.lib_variant));
     refreshNativeLogSetup();
-    return new mods[variant].LlamaContext(options);
+    const nativeCtx = new mods[variant].LlamaContext(options);
+    return new LlamaContextWrapper(nativeCtx);
 });
 exports.loadModel = loadModel;
 exports.initLlama = binding_1.loadModule;

package/lib/index.ts

@@ -1,5 +1,16 @@
 import { loadModule, LlamaModelOptions } from './binding'
-import type { Module, LlamaContext, LibVariant } from './binding'
+import type {
+  Module,
+  LlamaContext,
+  LibVariant,
+  ChatMessage,
+  LlamaCompletionOptions,
+  LlamaCompletionToken,
+  LlamaCompletionResult,
+  TokenizeResult,
+  EmbeddingResult,
+  CompletionResponseFormat,
+} from './binding'
 
 export * from './binding'
 
@@ -39,13 +50,202 @@ export function addNativeLogListener(
   }
 }
 
+const getJsonSchema = (responseFormat?: CompletionResponseFormat) => {
+  if (responseFormat?.type === 'json_schema') {
+    return responseFormat.json_schema?.schema
+  }
+  if (responseFormat?.type === 'json_object') {
+    return responseFormat.schema || {}
+  }
+  return null
+}
+
+class LlamaContextWrapper {
+  ctx: any
+
+  constructor(nativeCtx: any) {
+    this.ctx = nativeCtx
+  }
+
+  getSystemInfo(): string {
+    return this.ctx.getSystemInfo()
+  }
+
+  getModelInfo(): object {
+    return this.ctx.getModelInfo()
+  }
+
+  isJinjaSupported(): boolean {
+    const { minja } = this.ctx.getModelInfo().chatTemplates
+    return !!minja?.toolUse || !!minja?.default
+  }
+
+  isLlamaChatSupported(): boolean {
+    return !!this.ctx.getModelInfo().chatTemplates.llamaChat
+  }
+
+  _formatImageChat(messages: ChatMessage[] | undefined): {
+    messages: ChatMessage[] | undefined
+    has_image: boolean
+    image_paths?: string[]
+  } {
+    if (!messages)
+      return {
+        messages,
+        has_image: false,
+      }
+    const imagePaths: string[] = []
+    return {
+      messages: messages.map((msg) => {
+        if (Array.isArray(msg.content)) {
+          const content = msg.content.map((part) => {
+            // Handle multimodal content
+            if (part.type === 'image_url') {
+              let path = part.image_url?.url || ''
+              imagePaths.push(path)
+              return {
+                type: 'text',
+                text: '<__image__>',
+              }
+            }
+            return part
+          })
+
+          return {
+            ...msg,
+            content,
+          }
+        }
+        return msg
+      }),
+      has_image: imagePaths.length > 0,
+      image_paths: imagePaths,
+    }
+  }
+
+  getFormattedChat(
+    messages: ChatMessage[],
+    template?: string,
+    params?: {
+      jinja?: boolean
+      response_format?: CompletionResponseFormat
+      tools?: object
+      parallel_tool_calls?: object
+      tool_choice?: string
+    },
+  ): object {
+    const {
+      messages: chat,
+      has_image,
+      image_paths,
+    } = this._formatImageChat(messages)
+
+    const useJinja = this.isJinjaSupported() && params?.jinja
+    let tmpl = this.isLlamaChatSupported() || useJinja ? undefined : 'chatml'
+    if (template) tmpl = template // Force replace if provided
+    const jsonSchema = getJsonSchema(params?.response_format)
+
+    const result = this.ctx.getFormattedChat(chat, tmpl, {
+      jinja: useJinja,
+      json_schema: jsonSchema,
+      tools: params?.tools,
+      parallel_tool_calls: params?.parallel_tool_calls,
+      tool_choice: params?.tool_choice,
+    })
+
+    if (!useJinja) {
+      return {
+        type: 'llama-chat',
+        prompt: result as string,
+        has_image,
+        image_paths,
+      }
+    }
+    const jinjaResult = result
+    jinjaResult.type = 'jinja'
+    jinjaResult.has_image = has_image
+    jinjaResult.image_paths = image_paths
+    return jinjaResult
+  }
+
+  completion(
+    options: LlamaCompletionOptions,
+    callback?: (token: LlamaCompletionToken) => void,
+  ): Promise<LlamaCompletionResult> {
+    const { messages, image_paths = options.image_paths } =
+      this._formatImageChat(options.messages)
+    return this.ctx.completion({
+      ...options,
+      messages,
+      image_paths: options.image_paths || image_paths,
+    }, callback || (() => {}))
+  }
+
+  stopCompletion(): void {
+    return this.ctx.stopCompletion()
+  }
+
+  tokenize(text: string): Promise<TokenizeResult> {
+    return this.ctx.tokenize(text)
+  }
+
+  detokenize(tokens: number[]): Promise<string> {
+    return this.ctx.detokenize(tokens)
+  }
+
+  embedding(text: string): Promise<EmbeddingResult> {
+    return this.ctx.embedding(text)
+  }
+
+  saveSession(path: string): Promise<void> {
+    return this.ctx.saveSession(path)
+  }
+
+  loadSession(path: string): Promise<void> {
+    return this.ctx.loadSession(path)
+  }
+
+  release(): Promise<void> {
+    return this.ctx.release()
+  }
+
+  applyLoraAdapters(adapters: { path: string; scaled: number }[]): void {
+    return this.ctx.applyLoraAdapters(adapters)
+  }
+
+  removeLoraAdapters(adapters: { path: string }[]): void {
+    return this.ctx.removeLoraAdapters(adapters)
+  }
+
+  getLoadedLoraAdapters(): { path: string; scaled: number }[] {
+    return this.ctx.getLoadedLoraAdapters()
+  }
+
+  initMultimodal(options: {
+    path: string
+    use_gpu?: boolean
+  }): Promise<boolean> {
+    return this.ctx.initMultimodal(options)
+  }
+
+  isMultimodalEnabled(): Promise<boolean> {
+    return this.ctx.isMultimodalEnabled()
+  }
+
+  releaseMultimodal(): Promise<void> {
+    return this.ctx.releaseMultimodal()
+  }
+}
+
 export const loadModel = async (
   options: LlamaModelOptionsExtended,
-): Promise<LlamaContext> => {
+): Promise<LlamaContextWrapper> => {
   const variant = options.lib_variant ?? 'default'
   mods[variant] ??= await loadModule(options.lib_variant)
   refreshNativeLogSetup()
-  return new mods[variant].LlamaContext(options)
+
+  const nativeCtx = new mods[variant].LlamaContext(options)
+  return new LlamaContextWrapper(nativeCtx)
 }
 
 export const initLlama = loadModule

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.3.16",
+  "version": "0.4.0",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
+    "postinstall": "node scripts/bootstrap.js",
     "test": "jest",
     "build": "tsc",
     "prepack": "yarn build",

package/src/EmbeddingWorker.cpp

@@ -6,7 +6,7 @@ EmbeddingWorker::EmbeddingWorker(const Napi::CallbackInfo &info,
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text), _params(params) {}
 
 void EmbeddingWorker::Execute() {
-  llama_kv_cache_clear(_sess->context());
+  llama_kv_self_clear(_sess->context());
   auto tokens = ::common_tokenize(_sess->context(), _text, true);
   // add SEP if not present
   auto vocab = llama_model_get_vocab(_sess->model());