@fugood/llama.node 1.3.0-rc.2 → 1.3.0-rc.4

package/README.md

@@ -37,11 +37,9 @@ import { loadModel } from '@fugood/llama.node'
 // Initial a Llama context with the model (may take a while)
 const context = await loadModel({
   model: 'path/to/gguf/model',
-  use_mlock: true,
   n_ctx: 2048,
-  n_gpu_layers: 1, // > 0: enable GPU
-  // embedding: true, // use embedding
-  // lib_variant: 'opencl', // Change backend
+  n_gpu_layers: 99, // > 0: enable GPU
+  // lib_variant: 'vulkan', // Change backend
 })
 
 // Do completion

package/lib/binding.js

@@ -42,7 +42,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadModule = void 0;
+exports.isLibVariantAvailable = exports.loadModule = void 0;
 const getPlatformPackageName = (variant) => {
     const platform = process.platform;
     const arch = process.arch;
@@ -72,3 +72,21 @@ const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
     return (yield Promise.resolve().then(() => __importStar(require('../build/Release/index.node'))));
 });
 exports.loadModule = loadModule;
+const isLibVariantAvailable = (variant) => __awaiter(void 0, void 0, void 0, function* () {
+    if (variant && variant !== 'default') {
+        const module = yield loadPlatformPackage(getPlatformPackageName(variant));
+        return module != null;
+    }
+    const defaultModule = yield loadPlatformPackage(getPlatformPackageName());
+    if (defaultModule)
+        return true;
+    try {
+        // @ts-ignore
+        yield Promise.resolve().then(() => __importStar(require('../build/Release/index.node')));
+        return true;
+    }
+    catch (error) {
+        return false;
+    }
+});
+exports.isLibVariantAvailable = isLibVariantAvailable;

package/lib/binding.ts

@@ -587,3 +587,21 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
   // @ts-ignore
   return (await import('../build/Release/index.node')) as Module
 }
+
+export const isLibVariantAvailable = async (variant?: LibVariant): Promise<boolean> => {
+  if (variant && variant !== 'default') {
+    const module = await loadPlatformPackage(getPlatformPackageName(variant))
+    return module != null
+  }
+
+  const defaultModule = await loadPlatformPackage(getPlatformPackageName())
+  if (defaultModule) return true
+
+  try {
+    // @ts-ignore
+    await import('../build/Release/index.node')
+    return true
+  } catch (error) {
+    return false
+  }
+}

package/lib/index.js

@@ -23,14 +23,14 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BuildInfo = exports.getBackendDevicesInfo = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.MTMD_DEFAULT_MEDIA_MARKER = exports.LlamaParallelAPI = void 0;
+exports.BuildInfo = exports.getBackendDevicesInfo = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.LlamaParallelAPI = void 0;
 exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
 const version_1 = require("./version");
 const parallel_1 = require("./parallel");
 Object.defineProperty(exports, "LlamaParallelAPI", { enumerable: true, get: function () { return parallel_1.LlamaParallelAPI; } });
+const utils_1 = require("./utils");
 __exportStar(require("./binding"), exports);
-exports.MTMD_DEFAULT_MEDIA_MARKER = '<__media__>';
 const mods = {};
 const logListeners = [];
 const logCallback = (level, text) => {
@@ -83,60 +83,9 @@ class LlamaContextWrapper {
     isLlamaChatSupported() {
         return !!this.ctx.getModelInfo().chatTemplates.llamaChat;
     }
-    _formatMediaChat(messages) {
-        if (!messages)
-            return {
-                messages,
-                has_media: false,
-            };
-        const mediaPaths = [];
-        return {
-            messages: messages.map((msg) => {
-                if (Array.isArray(msg.content)) {
-                    const content = msg.content.map((part) => {
-                        var _a;
-                        // Handle multimodal content
-                        if (part.type === 'image_url') {
-                            let path = ((_a = part.image_url) === null || _a === void 0 ? void 0 : _a.url) || '';
-                            mediaPaths.push(path);
-                            return {
-                                type: 'text',
-                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
-                            };
-                        }
-                        else if (part.type === 'input_audio') {
-                            const { input_audio: audio } = part;
-                            if (!audio)
-                                throw new Error('input_audio is required');
-                            const { format } = audio;
-                            if (format != 'wav' && format != 'mp3') {
-                                throw new Error(`Unsupported audio format: ${format}`);
-                            }
-                            if (audio.url) {
-                                const path = audio.url.replace(/file:\/\//, '');
-                                mediaPaths.push(path);
-                            }
-                            else if (audio.data) {
-                                mediaPaths.push(audio.data);
-                            }
-                            return {
-                                type: 'text',
-                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
-                            };
-                        }
-                        return part;
-                    });
-                    return Object.assign(Object.assign({}, msg), { content });
-                }
-                return msg;
-            }),
-            has_media: mediaPaths.length > 0,
-            media_paths: mediaPaths,
-        };
-    }
     getFormattedChat(messages, template, params) {
         var _a;
-        const { messages: chat, has_media, media_paths, } = this._formatMediaChat(messages);
+        const { messages: chat, has_media, media_paths, } = (0, utils_1.formatMediaChat)(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
         if (template)
@@ -170,7 +119,7 @@ class LlamaContextWrapper {
             media_paths }, jinjaResult);
     }
     completion(options, callback) {
-        const { messages, media_paths = options.media_paths } = this._formatMediaChat(options.messages);
+        const { messages, media_paths = options.media_paths } = (0, utils_1.formatMediaChat)(options.messages);
         return this.ctx.completion(Object.assign(Object.assign({}, options), { messages, media_paths: options.media_paths || media_paths }), callback || (() => { }));
     }
     stopCompletion() {

package/lib/index.ts

@@ -19,12 +19,11 @@ import type {
 } from './binding'
 import { BUILD_NUMBER, BUILD_COMMIT } from './version'
 import { LlamaParallelAPI } from './parallel'
+import { formatMediaChat } from './utils'
 
 export * from './binding'
 export { LlamaParallelAPI }
 
-export const MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'
-
 export interface LlamaModelOptionsExtended extends LlamaModelOptions {
   lib_variant?: LibVariant
 }
@@ -104,63 +103,6 @@ class LlamaContextWrapper {
     return !!this.ctx.getModelInfo().chatTemplates.llamaChat
   }
 
-  _formatMediaChat(messages: ChatMessage[] | undefined): {
-    messages: ChatMessage[] | undefined
-    has_media: boolean
-    media_paths?: string[]
-  } {
-    if (!messages)
-      return {
-        messages,
-        has_media: false,
-      }
-    const mediaPaths: string[] = []
-    return {
-      messages: messages.map((msg) => {
-        if (Array.isArray(msg.content)) {
-          const content = msg.content.map((part) => {
-            // Handle multimodal content
-            if (part.type === 'image_url') {
-              let path = part.image_url?.url || ''
-              mediaPaths.push(path)
-              return {
-                type: 'text',
-                text: MTMD_DEFAULT_MEDIA_MARKER,
-              }
-            } else if (part.type === 'input_audio') {
-              const { input_audio: audio } = part
-              if (!audio) throw new Error('input_audio is required')
-
-              const { format } = audio
-              if (format != 'wav' && format != 'mp3') {
-                throw new Error(`Unsupported audio format: ${format}`)
-              }
-              if (audio.url) {
-                const path = audio.url.replace(/file:\/\//, '')
-                mediaPaths.push(path)
-              } else if (audio.data) {
-                mediaPaths.push(audio.data)
-              }
-              return {
-                type: 'text',
-                text: MTMD_DEFAULT_MEDIA_MARKER,
-              }
-            }
-            return part
-          })
-
-          return {
-            ...msg,
-            content,
-          }
-        }
-        return msg
-      }),
-      has_media: mediaPaths.length > 0,
-      media_paths: mediaPaths,
-    }
-  }
-
   getFormattedChat(
     messages: ChatMessage[],
     template?: string,
@@ -180,7 +122,7 @@ class LlamaContextWrapper {
       messages: chat,
       has_media,
       media_paths,
-    } = this._formatMediaChat(messages)
+    } = formatMediaChat(messages)
 
     const useJinja = this.isJinjaSupported() && params?.jinja
     let tmpl
@@ -228,7 +170,7 @@ class LlamaContextWrapper {
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult> {
     const { messages, media_paths = options.media_paths } =
-      this._formatMediaChat(options.messages)
+      formatMediaChat(options.messages)
     return this.ctx.completion(
       {
         ...options,

package/lib/parallel.js

@@ -10,6 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LlamaParallelAPI = void 0;
+const utils_1 = require("./utils");
 class LlamaParallelAPI {
     constructor(context) {
         this.enabled = false;
@@ -87,8 +88,9 @@ class LlamaParallelAPI {
                     }
                 }
                 : undefined;
+            const { messages, media_paths = options.media_paths } = (0, utils_1.formatMediaChat)(options.messages);
             // Queue the completion immediately (this is synchronous!)
-            const { requestId } = this.context.queueCompletion(options, tokenCallback ||
+            const { requestId } = this.context.queueCompletion(Object.assign(Object.assign({}, options), { messages, media_paths: media_paths }), tokenCallback ||
                 ((error, result) => {
                     if (error) {
                         const pendingReq = this.pendingRequests.get(result === null || result === void 0 ? void 0 : result.requestId);

package/lib/parallel.ts

@@ -5,6 +5,7 @@ import type {
   LlamaCompletionToken,
   RerankParams,
 } from './binding'
+import { formatMediaChat } from './utils'
 
 export class LlamaParallelAPI {
   private context: LlamaContext
@@ -109,9 +110,16 @@ export class LlamaParallelAPI {
         }
       : undefined
 
+    const { messages, media_paths = options.media_paths } = formatMediaChat(
+      options.messages,
+    )
     // Queue the completion immediately (this is synchronous!)
     const { requestId } = this.context.queueCompletion(
-      options,
+      {
+        ...options,
+        messages,
+        media_paths: media_paths,
+      },
       tokenCallback ||
         ((error, result) => {
           if (error) {

package/lib/utils.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatMediaChat = exports.MTMD_DEFAULT_MEDIA_MARKER = void 0;
+exports.MTMD_DEFAULT_MEDIA_MARKER = '<__media__>';
+const formatMediaChat = (messages) => {
+    if (!messages)
+        return {
+            messages,
+            has_media: false,
+        };
+    const mediaPaths = [];
+    return {
+        messages: messages.map((msg) => {
+            if (Array.isArray(msg.content)) {
+                const content = msg.content.map((part) => {
+                    var _a;
+                    // Handle multimodal content
+                    if (part.type === 'image_url') {
+                        let path = ((_a = part.image_url) === null || _a === void 0 ? void 0 : _a.url) || '';
+                        mediaPaths.push(path);
+                        return {
+                            type: 'text',
+                            text: exports.MTMD_DEFAULT_MEDIA_MARKER,
+                        };
+                    }
+                    else if (part.type === 'input_audio') {
+                        const { input_audio: audio } = part;
+                        if (!audio)
+                            throw new Error('input_audio is required');
+                        const { format } = audio;
+                        if (format != 'wav' && format != 'mp3') {
+                            throw new Error(`Unsupported audio format: ${format}`);
+                        }
+                        if (audio.url) {
+                            const path = audio.url.replace(/file:\/\//, '');
+                            mediaPaths.push(path);
+                        }
+                        else if (audio.data) {
+                            mediaPaths.push(audio.data);
+                        }
+                        return {
+                            type: 'text',
+                            text: exports.MTMD_DEFAULT_MEDIA_MARKER,
+                        };
+                    }
+                    return part;
+                });
+                return Object.assign(Object.assign({}, msg), { content });
+            }
+            return msg;
+        }),
+        has_media: mediaPaths.length > 0,
+        media_paths: mediaPaths,
+    };
+};
+exports.formatMediaChat = formatMediaChat;

package/lib/utils.ts

@@ -0,0 +1,63 @@
+
+import type {
+  ChatMessage,
+} from './binding'
+
+export const MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'
+
+export const formatMediaChat = (messages: ChatMessage[] | undefined): {
+  messages: ChatMessage[] | undefined
+  has_media: boolean
+  media_paths?: string[]
+} => {
+  if (!messages)
+    return {
+      messages,
+      has_media: false,
+    }
+  const mediaPaths: string[] = []
+  return {
+    messages: messages.map((msg) => {
+      if (Array.isArray(msg.content)) {
+        const content = msg.content.map((part) => {
+          // Handle multimodal content
+          if (part.type === 'image_url') {
+            let path = part.image_url?.url || ''
+            mediaPaths.push(path)
+            return {
+              type: 'text',
+              text: MTMD_DEFAULT_MEDIA_MARKER,
+            }
+          } else if (part.type === 'input_audio') {
+            const { input_audio: audio } = part
+            if (!audio) throw new Error('input_audio is required')
+
+            const { format } = audio
+            if (format != 'wav' && format != 'mp3') {
+              throw new Error(`Unsupported audio format: ${format}`)
+            }
+            if (audio.url) {
+              const path = audio.url.replace(/file:\/\//, '')
+              mediaPaths.push(path)
+            } else if (audio.data) {
+              mediaPaths.push(audio.data)
+            }
+            return {
+              type: 'text',
+              text: MTMD_DEFAULT_MEDIA_MARKER,
+            }
+          }
+          return part
+        })
+
+        return {
+          ...msg,
+          content,
+        }
+      }
+      return msg
+    }),
+    has_media: mediaPaths.length > 0,
+    media_paths: mediaPaths,
+  }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.3.0-rc.2",
+  "version": "1.3.0-rc.4",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,19 +72,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.3.0-rc.2",
-    "@fugood/node-llama-linux-x64-vulkan": "1.3.0-rc.2",
-    "@fugood/node-llama-linux-x64-cuda": "1.3.0-rc.2",
-    "@fugood/node-llama-linux-arm64": "1.3.0-rc.2",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.3.0-rc.2",
-    "@fugood/node-llama-linux-arm64-cuda": "1.3.0-rc.2",
-    "@fugood/node-llama-win32-x64": "1.3.0-rc.2",
-    "@fugood/node-llama-win32-x64-vulkan": "1.3.0-rc.2",
-    "@fugood/node-llama-win32-x64-cuda": "1.3.0-rc.2",
-    "@fugood/node-llama-win32-arm64": "1.3.0-rc.2",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.3.0-rc.2",
-    "@fugood/node-llama-darwin-x64": "1.3.0-rc.2",
-    "@fugood/node-llama-darwin-arm64": "1.3.0-rc.2"
+    "@fugood/node-llama-linux-x64": "1.3.0-rc.4",
+    "@fugood/node-llama-linux-x64-vulkan": "1.3.0-rc.4",
+    "@fugood/node-llama-linux-x64-cuda": "1.3.0-rc.4",
+    "@fugood/node-llama-linux-arm64": "1.3.0-rc.4",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.3.0-rc.4",
+    "@fugood/node-llama-linux-arm64-cuda": "1.3.0-rc.4",
+    "@fugood/node-llama-win32-x64": "1.3.0-rc.4",
+    "@fugood/node-llama-win32-x64-vulkan": "1.3.0-rc.4",
+    "@fugood/node-llama-win32-x64-cuda": "1.3.0-rc.4",
+    "@fugood/node-llama-win32-arm64": "1.3.0-rc.4",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.3.0-rc.4",
+    "@fugood/node-llama-darwin-x64": "1.3.0-rc.4",
+    "@fugood/node-llama-darwin-arm64": "1.3.0-rc.4"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",