@fugood/llama.node 1.3.0-rc.1 → 1.3.0-rc.4

package/README.md

@@ -37,11 +37,9 @@ import { loadModel } from '@fugood/llama.node'
 // Initial a Llama context with the model (may take a while)
 const context = await loadModel({
   model: 'path/to/gguf/model',
-  use_mlock: true,
   n_ctx: 2048,
-  n_gpu_layers: 1, // > 0: enable GPU
-  // embedding: true, // use embedding
-  // lib_variant: 'opencl', // Change backend
+  n_gpu_layers: 99, // > 0: enable GPU
+  // lib_variant: 'vulkan', // Change backend
 })
 
 // Do completion

package/lib/binding.js

@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -32,7 +42,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadModule = void 0;
+exports.isLibVariantAvailable = exports.loadModule = void 0;
 const getPlatformPackageName = (variant) => {
     const platform = process.platform;
     const arch = process.arch;
@@ -62,3 +72,21 @@ const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
     return (yield Promise.resolve().then(() => __importStar(require('../build/Release/index.node'))));
 });
 exports.loadModule = loadModule;
+const isLibVariantAvailable = (variant) => __awaiter(void 0, void 0, void 0, function* () {
+    if (variant && variant !== 'default') {
+        const module = yield loadPlatformPackage(getPlatformPackageName(variant));
+        return module != null;
+    }
+    const defaultModule = yield loadPlatformPackage(getPlatformPackageName());
+    if (defaultModule)
+        return true;
+    try {
+        // @ts-ignore
+        yield Promise.resolve().then(() => __importStar(require('../build/Release/index.node')));
+        return true;
+    }
+    catch (error) {
+        return false;
+    }
+});
+exports.isLibVariantAvailable = isLibVariantAvailable;

package/lib/binding.ts

@@ -587,3 +587,21 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
   // @ts-ignore
   return (await import('../build/Release/index.node')) as Module
 }
+
+export const isLibVariantAvailable = async (variant?: LibVariant): Promise<boolean> => {
+  if (variant && variant !== 'default') {
+    const module = await loadPlatformPackage(getPlatformPackageName(variant))
+    return module != null
+  }
+
+  const defaultModule = await loadPlatformPackage(getPlatformPackageName())
+  if (defaultModule) return true
+
+  try {
+    // @ts-ignore
+    await import('../build/Release/index.node')
+    return true
+  } catch (error) {
+    return false
+  }
+}

package/lib/index.js

@@ -23,14 +23,14 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BuildInfo = exports.getBackendDevicesInfo = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.MTMD_DEFAULT_MEDIA_MARKER = exports.LlamaParallelAPI = void 0;
+exports.BuildInfo = exports.getBackendDevicesInfo = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.LlamaParallelAPI = void 0;
 exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
 const version_1 = require("./version");
 const parallel_1 = require("./parallel");
 Object.defineProperty(exports, "LlamaParallelAPI", { enumerable: true, get: function () { return parallel_1.LlamaParallelAPI; } });
+const utils_1 = require("./utils");
 __exportStar(require("./binding"), exports);
-exports.MTMD_DEFAULT_MEDIA_MARKER = '<__media__>';
 const mods = {};
 const logListeners = [];
 const logCallback = (level, text) => {
@@ -83,60 +83,9 @@ class LlamaContextWrapper {
     isLlamaChatSupported() {
         return !!this.ctx.getModelInfo().chatTemplates.llamaChat;
     }
-    _formatMediaChat(messages) {
-        if (!messages)
-            return {
-                messages,
-                has_media: false,
-            };
-        const mediaPaths = [];
-        return {
-            messages: messages.map((msg) => {
-                if (Array.isArray(msg.content)) {
-                    const content = msg.content.map((part) => {
-                        var _a;
-                        // Handle multimodal content
-                        if (part.type === 'image_url') {
-                            let path = ((_a = part.image_url) === null || _a === void 0 ? void 0 : _a.url) || '';
-                            mediaPaths.push(path);
-                            return {
-                                type: 'text',
-                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
-                            };
-                        }
-                        else if (part.type === 'input_audio') {
-                            const { input_audio: audio } = part;
-                            if (!audio)
-                                throw new Error('input_audio is required');
-                            const { format } = audio;
-                            if (format != 'wav' && format != 'mp3') {
-                                throw new Error(`Unsupported audio format: ${format}`);
-                            }
-                            if (audio.url) {
-                                const path = audio.url.replace(/file:\/\//, '');
-                                mediaPaths.push(path);
-                            }
-                            else if (audio.data) {
-                                mediaPaths.push(audio.data);
-                            }
-                            return {
-                                type: 'text',
-                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
-                            };
-                        }
-                        return part;
-                    });
-                    return Object.assign(Object.assign({}, msg), { content });
-                }
-                return msg;
-            }),
-            has_media: mediaPaths.length > 0,
-            media_paths: mediaPaths,
-        };
-    }
     getFormattedChat(messages, template, params) {
         var _a;
-        const { messages: chat, has_media, media_paths, } = this._formatMediaChat(messages);
+        const { messages: chat, has_media, media_paths, } = (0, utils_1.formatMediaChat)(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
         if (template)
@@ -170,7 +119,7 @@ class LlamaContextWrapper {
             media_paths }, jinjaResult);
     }
     completion(options, callback) {
-        const { messages, media_paths = options.media_paths } = this._formatMediaChat(options.messages);
+        const { messages, media_paths = options.media_paths } = (0, utils_1.formatMediaChat)(options.messages);
         return this.ctx.completion(Object.assign(Object.assign({}, options), { messages, media_paths: options.media_paths || media_paths }), callback || (() => { }));
     }
     stopCompletion() {

package/lib/index.ts

@@ -19,12 +19,11 @@ import type {
 } from './binding'
 import { BUILD_NUMBER, BUILD_COMMIT } from './version'
 import { LlamaParallelAPI } from './parallel'
+import { formatMediaChat } from './utils'
 
 export * from './binding'
 export { LlamaParallelAPI }
 
-export const MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'
-
 export interface LlamaModelOptionsExtended extends LlamaModelOptions {
   lib_variant?: LibVariant
 }
@@ -104,63 +103,6 @@ class LlamaContextWrapper {
     return !!this.ctx.getModelInfo().chatTemplates.llamaChat
   }
 
-  _formatMediaChat(messages: ChatMessage[] | undefined): {
-    messages: ChatMessage[] | undefined
-    has_media: boolean
-    media_paths?: string[]
-  } {
-    if (!messages)
-      return {
-        messages,
-        has_media: false,
-      }
-    const mediaPaths: string[] = []
-    return {
-      messages: messages.map((msg) => {
-        if (Array.isArray(msg.content)) {
-          const content = msg.content.map((part) => {
-            // Handle multimodal content
-            if (part.type === 'image_url') {
-              let path = part.image_url?.url || ''
-              mediaPaths.push(path)
-              return {
-                type: 'text',
-                text: MTMD_DEFAULT_MEDIA_MARKER,
-              }
-            } else if (part.type === 'input_audio') {
-              const { input_audio: audio } = part
-              if (!audio) throw new Error('input_audio is required')
-
-              const { format } = audio
-              if (format != 'wav' && format != 'mp3') {
-                throw new Error(`Unsupported audio format: ${format}`)
-              }
-              if (audio.url) {
-                const path = audio.url.replace(/file:\/\//, '')
-                mediaPaths.push(path)
-              } else if (audio.data) {
-                mediaPaths.push(audio.data)
-              }
-              return {
-                type: 'text',
-                text: MTMD_DEFAULT_MEDIA_MARKER,
-              }
-            }
-            return part
-          })
-
-          return {
-            ...msg,
-            content,
-          }
-        }
-        return msg
-      }),
-      has_media: mediaPaths.length > 0,
-      media_paths: mediaPaths,
-    }
-  }
-
   getFormattedChat(
     messages: ChatMessage[],
     template?: string,
@@ -180,7 +122,7 @@ class LlamaContextWrapper {
       messages: chat,
       has_media,
       media_paths,
-    } = this._formatMediaChat(messages)
+    } = formatMediaChat(messages)
 
     const useJinja = this.isJinjaSupported() && params?.jinja
     let tmpl
@@ -228,7 +170,7 @@ class LlamaContextWrapper {
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult> {
     const { messages, media_paths = options.media_paths } =
-      this._formatMediaChat(options.messages)
+      formatMediaChat(options.messages)
     return this.ctx.completion(
       {
         ...options,

package/lib/parallel.js

@@ -10,6 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LlamaParallelAPI = void 0;
+const utils_1 = require("./utils");
 class LlamaParallelAPI {
     constructor(context) {
         this.enabled = false;
@@ -87,8 +88,9 @@ class LlamaParallelAPI {
                     }
                 }
                 : undefined;
+            const { messages, media_paths = options.media_paths } = (0, utils_1.formatMediaChat)(options.messages);
             // Queue the completion immediately (this is synchronous!)
-            const { requestId } = this.context.queueCompletion(options, tokenCallback ||
+            const { requestId } = this.context.queueCompletion(Object.assign(Object.assign({}, options), { messages, media_paths: media_paths }), tokenCallback ||
                 ((error, result) => {
                     if (error) {
                         const pendingReq = this.pendingRequests.get(result === null || result === void 0 ? void 0 : result.requestId);

package/lib/parallel.ts

@@ -5,6 +5,7 @@ import type {
   LlamaCompletionToken,
   RerankParams,
 } from './binding'
+import { formatMediaChat } from './utils'
 
 export class LlamaParallelAPI {
   private context: LlamaContext
@@ -109,9 +110,16 @@ export class LlamaParallelAPI {
         }
       : undefined
 
+    const { messages, media_paths = options.media_paths } = formatMediaChat(
+      options.messages,
+    )
     // Queue the completion immediately (this is synchronous!)
     const { requestId } = this.context.queueCompletion(
-      options,
+      {
+        ...options,
+        messages,
+        media_paths: media_paths,
+      },
       tokenCallback ||
         ((error, result) => {
           if (error) {

package/lib/utils.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatMediaChat = exports.MTMD_DEFAULT_MEDIA_MARKER = void 0;
+exports.MTMD_DEFAULT_MEDIA_MARKER = '<__media__>';
+const formatMediaChat = (messages) => {
+    if (!messages)
+        return {
+            messages,
+            has_media: false,
+        };
+    const mediaPaths = [];
+    return {
+        messages: messages.map((msg) => {
+            if (Array.isArray(msg.content)) {
+                const content = msg.content.map((part) => {
+                    var _a;
+                    // Handle multimodal content
+                    if (part.type === 'image_url') {
+                        let path = ((_a = part.image_url) === null || _a === void 0 ? void 0 : _a.url) || '';
+                        mediaPaths.push(path);
+                        return {
+                            type: 'text',
+                            text: exports.MTMD_DEFAULT_MEDIA_MARKER,
+                        };
+                    }
+                    else if (part.type === 'input_audio') {
+                        const { input_audio: audio } = part;
+                        if (!audio)
+                            throw new Error('input_audio is required');
+                        const { format } = audio;
+                        if (format != 'wav' && format != 'mp3') {
+                            throw new Error(`Unsupported audio format: ${format}`);
+                        }
+                        if (audio.url) {
+                            const path = audio.url.replace(/file:\/\//, '');
+                            mediaPaths.push(path);
+                        }
+                        else if (audio.data) {
+                            mediaPaths.push(audio.data);
+                        }
+                        return {
+                            type: 'text',
+                            text: exports.MTMD_DEFAULT_MEDIA_MARKER,
+                        };
+                    }
+                    return part;
+                });
+                return Object.assign(Object.assign({}, msg), { content });
+            }
+            return msg;
+        }),
+        has_media: mediaPaths.length > 0,
+        media_paths: mediaPaths,
+    };
+};
+exports.formatMediaChat = formatMediaChat;

package/lib/utils.ts

@@ -0,0 +1,63 @@
+
+import type {
+  ChatMessage,
+} from './binding'
+
+export const MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'
+
+export const formatMediaChat = (messages: ChatMessage[] | undefined): {
+  messages: ChatMessage[] | undefined
+  has_media: boolean
+  media_paths?: string[]
+} => {
+  if (!messages)
+    return {
+      messages,
+      has_media: false,
+    }
+  const mediaPaths: string[] = []
+  return {
+    messages: messages.map((msg) => {
+      if (Array.isArray(msg.content)) {
+        const content = msg.content.map((part) => {
+          // Handle multimodal content
+          if (part.type === 'image_url') {
+            let path = part.image_url?.url || ''
+            mediaPaths.push(path)
+            return {
+              type: 'text',
+              text: MTMD_DEFAULT_MEDIA_MARKER,
+            }
+          } else if (part.type === 'input_audio') {
+            const { input_audio: audio } = part
+            if (!audio) throw new Error('input_audio is required')
+
+            const { format } = audio
+            if (format != 'wav' && format != 'mp3') {
+              throw new Error(`Unsupported audio format: ${format}`)
+            }
+            if (audio.url) {
+              const path = audio.url.replace(/file:\/\//, '')
+              mediaPaths.push(path)
+            } else if (audio.data) {
+              mediaPaths.push(audio.data)
+            }
+            return {
+              type: 'text',
+              text: MTMD_DEFAULT_MEDIA_MARKER,
+            }
+          }
+          return part
+        })
+
+        return {
+          ...msg,
+          content,
+        }
+      }
+      return msg
+    }),
+    has_media: mediaPaths.length > 0,
+    media_paths: mediaPaths,
+  }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.3.0-rc.1",
+  "version": "1.3.0-rc.4",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,19 +72,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.3.0-rc.1",
-    "@fugood/node-llama-linux-x64-vulkan": "1.3.0-rc.1",
-    "@fugood/node-llama-linux-x64-cuda": "1.3.0-rc.1",
-    "@fugood/node-llama-linux-arm64": "1.3.0-rc.1",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.3.0-rc.1",
-    "@fugood/node-llama-linux-arm64-cuda": "1.3.0-rc.1",
-    "@fugood/node-llama-win32-x64": "1.3.0-rc.1",
-    "@fugood/node-llama-win32-x64-vulkan": "1.3.0-rc.1",
-    "@fugood/node-llama-win32-x64-cuda": "1.3.0-rc.1",
-    "@fugood/node-llama-win32-arm64": "1.3.0-rc.1",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.3.0-rc.1",
-    "@fugood/node-llama-darwin-x64": "1.3.0-rc.1",
-    "@fugood/node-llama-darwin-arm64": "1.3.0-rc.1"
+    "@fugood/node-llama-linux-x64": "1.3.0-rc.4",
+    "@fugood/node-llama-linux-x64-vulkan": "1.3.0-rc.4",
+    "@fugood/node-llama-linux-x64-cuda": "1.3.0-rc.4",
+    "@fugood/node-llama-linux-arm64": "1.3.0-rc.4",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.3.0-rc.4",
+    "@fugood/node-llama-linux-arm64-cuda": "1.3.0-rc.4",
+    "@fugood/node-llama-win32-x64": "1.3.0-rc.4",
+    "@fugood/node-llama-win32-x64-vulkan": "1.3.0-rc.4",
+    "@fugood/node-llama-win32-x64-cuda": "1.3.0-rc.4",
+    "@fugood/node-llama-win32-arm64": "1.3.0-rc.4",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.3.0-rc.4",
+    "@fugood/node-llama-darwin-x64": "1.3.0-rc.4",
+    "@fugood/node-llama-darwin-arm64": "1.3.0-rc.4"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/src/llama.cpp/common/json-schema-to-grammar.cpp

@@ -41,9 +41,9 @@ static std::string build_repetition(const std::string & item_rule, int min_items
     return result;
 }
 
-static void _build_min_max_int(int min_value, int max_value, std::stringstream & out, int decimals_left = 16, bool top_level = true) {
-    auto has_min = min_value != std::numeric_limits<int>::min();
-    auto has_max = max_value != std::numeric_limits<int>::max();
+static void _build_min_max_int(int64_t min_value, int64_t max_value, std::stringstream & out, int decimals_left = 16, bool top_level = true) {
+    auto has_min = min_value != std::numeric_limits<int64_t>::min();
+    auto has_max = max_value != std::numeric_limits<int64_t>::max();
 
     auto digit_range = [&](char from, char to) {
         out << "[";
@@ -159,7 +159,7 @@ static void _build_min_max_int(int min_value, int max_value, std::stringstream &
     if (has_min) {
         if (min_value < 0) {
             out << "\"-\" (";
-            _build_min_max_int(std::numeric_limits<int>::min(), -min_value, out, decimals_left, /* top_level= */ false);
+            _build_min_max_int(std::numeric_limits<int64_t>::min(), -min_value, out, decimals_left, /* top_level= */ false);
             out << ") | [0] | [1-9] ";
             more_digits(0, decimals_left - 1);
         } else if (min_value == 0) {
@@ -194,7 +194,7 @@ static void _build_min_max_int(int min_value, int max_value, std::stringstream &
             }
             digit_range(c, c);
             out << " (";
-            _build_min_max_int(std::stoi(min_s.substr(1)), std::numeric_limits<int>::max(), out, less_decimals, /* top_level= */ false);
+            _build_min_max_int(std::stoll(min_s.substr(1)), std::numeric_limits<int64_t>::max(), out, less_decimals, /* top_level= */ false);
             out << ")";
             if (c < '9') {
                 out << " | ";
@@ -216,7 +216,7 @@ static void _build_min_max_int(int min_value, int max_value, std::stringstream &
             _build_min_max_int(0, max_value, out, decimals_left, /* top_level= */ true);
         } else {
             out << "\"-\" (";
-            _build_min_max_int(-max_value, std::numeric_limits<int>::max(), out, decimals_left, /* top_level= */ false);
+            _build_min_max_int(-max_value, std::numeric_limits<int64_t>::max(), out, decimals_left, /* top_level= */ false);
             out << ")";
         }
         return;
@@ -925,17 +925,17 @@ public:
             int max_len = schema.contains("maxLength") ? schema["maxLength"].get<int>() : std::numeric_limits<int>::max();
             return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space");
         } else if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) {
-            int min_value = std::numeric_limits<int>::min();
-            int max_value = std::numeric_limits<int>::max();
+            int64_t min_value = std::numeric_limits<int64_t>::min();
+            int64_t max_value = std::numeric_limits<int64_t>::max();
             if (schema.contains("minimum")) {
-                min_value = schema["minimum"].get<int>();
+                min_value = schema["minimum"].get<int64_t>();
             } else if (schema.contains("exclusiveMinimum")) {
-                min_value = schema["exclusiveMinimum"].get<int>() + 1;
+                min_value = schema["exclusiveMinimum"].get<int64_t>() + 1;
             }
             if (schema.contains("maximum")) {
-                max_value = schema["maximum"].get<int>();
+                max_value = schema["maximum"].get<int64_t>();
             } else if (schema.contains("exclusiveMaximum")) {
-                max_value = schema["exclusiveMaximum"].get<int>() - 1;
+                max_value = schema["exclusiveMaximum"].get<int64_t>() - 1;
             }
             std::stringstream out;
             out << "(";

package/src/llama.cpp/ggml/include/ggml-rpc.h

@@ -21,8 +21,7 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const c
 GGML_BACKEND_API void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total);
 
 GGML_BACKEND_API void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir,
-                                                    size_t n_threads, size_t n_devices,
-                                                    ggml_backend_dev_t * devices, size_t * free_mem, size_t * total_mem);
+                                                    size_t n_threads, size_t n_devices, ggml_backend_dev_t * devices);
 
 GGML_BACKEND_API ggml_backend_reg_t ggml_backend_rpc_reg(void);
 GGML_BACKEND_API ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint);

package/src/llama.cpp/ggml/src/ggml-cpu/spacemit/ime.cpp

@@ -485,8 +485,9 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS> class tensor_
             int32_t          start                  = ith * task_per_thread;
             int32_t          end                    = std::min((ith + 1) * task_per_thread, task_count);
             for (int32_t compute_idx = start; compute_idx < end; compute_idx++) {
-                int32_t                             gemm_idx = compute_idx / block_size_m;
-                int32_t                             m_idx    = compute_idx % block_size_m * block_size_m;
+                int32_t                             gemm_idx = compute_idx / per_gemm_block_count_m;
+                int32_t                             block_idx_in_gemm = compute_idx % per_gemm_block_count_m;
+                int32_t                             m_idx    = block_idx_in_gemm * block_size_m;
                 const qnbitgemm_spacemit_ime_args & data     = qnbitgemm_args[gemm_idx];
                 int32_t rows_tobe_handled = (gemm_m - m_idx) > block_size_m ? block_size_m : (gemm_m - m_idx);
 

package/src/llama.cpp/src/llama-model.cpp

@@ -421,11 +421,8 @@ struct llama_model::impl {
     llama_mlocks mlock_bufs;
     llama_mlocks mlock_mmaps;
 
-    // contexts where the model tensors metadata is stored
-    std::vector<ggml_context_ptr> ctxs;
-
-    // the model memory buffers for the tensor data
-    std::vector<ggml_backend_buffer_ptr> bufs;
+    // contexts where the model tensors metadata is stored as well ass the corresponding buffers:
+    std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
 
     buft_list_t cpu_buft_list;
     std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;
@@ -2182,7 +2179,14 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
     max_n_tensors += n_layer*2; // duplicated rope freq tensors
     const size_t ctx_size = ggml_tensor_overhead()*max_n_tensors;
 
-    std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;
+    // define a comparator for the buft -> ctx map to ensure that the order is well-defined:
+    struct ggml_backend_buft_comparator {
+        bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const {
+            return ggml_backend_buft_name(lhs) < ggml_backend_buft_name(rhs);
+        }
+    };
+    std::map<ggml_backend_buffer_type_t, ggml_context_ptr, ggml_backend_buft_comparator> ctx_map;
+
     auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {
         auto it = ctx_map.find(buft);
         if (it == ctx_map.end()) {
@@ -2197,12 +2201,11 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
                 throw std::runtime_error(format("failed to create ggml context"));
             }
 
-            ctx_map[buft] = ctx;
-            pimpl->ctxs.emplace_back(ctx);
+            ctx_map.emplace(buft, ctx);
 
             return ctx;
         }
-        return it->second;
+        return it->second.get();
     };
 
     const auto TENSOR_DUPLICATED   = llama_model_loader::TENSOR_DUPLICATED;
@@ -6037,16 +6040,15 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
     pimpl->mappings.reserve(ml.mappings.size());
 
     // create the backend buffers
-    std::vector<std::pair<ggml_context *, llama_buf_map>> ctx_bufs;
-    ctx_bufs.reserve(ctx_map.size());
+    std::vector<std::pair<ggml_context *, llama_buf_map>> ctx_buf_maps;
+    ctx_buf_maps.reserve(ctx_map.size());
 
     // Ensure we have enough capacity for the maximum backend buffer we will potentially create
     const size_t n_max_backend_buffer = ctx_map.size() * ml.files.size();
-    pimpl->bufs.reserve(n_max_backend_buffer);
+    pimpl->ctxs_bufs.reserve(n_max_backend_buffer);
 
-    for (auto & it : ctx_map) {
-        ggml_backend_buffer_type_t buft = it.first;
-        ggml_context * ctx              = it.second;
+    for (auto & [buft, ctx_ptr] : ctx_map) {
+        ggml_context * ctx = ctx_ptr.get();
 
         // skip contexts without tensors
         if (ggml_get_first_tensor(ctx) == nullptr) {
@@ -6070,6 +6072,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
         bool buffer_from_host_ptr_supported = props.caps.buffer_from_host_ptr;
         bool is_default_buft = buft == ggml_backend_dev_buffer_type(dev);
 
+        ggml_backend_buffer_t buf = nullptr;
         if (ml.use_mmap && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {
             for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
                 // only the mmap region containing the tensors in the model is mapped to the backend buffer
@@ -6082,20 +6085,18 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
                     continue;
                 }
                 const size_t max_size = ggml_get_max_tensor_size(ctx);
-                ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + first, last - first, max_size);
+                buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + first, last - first, max_size);
                 if (buf == nullptr) {
                     throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));
                 }
-                pimpl->bufs.emplace_back(buf);
                 buf_map.emplace(idx, buf);
             }
         }
         else {
-            ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);
+            buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);
             if (buf == nullptr) {
                 throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));
             }
-            pimpl->bufs.emplace_back(buf);
             if (use_mlock && ggml_backend_buffer_is_host(buf)) {
                 pimpl->mlock_bufs.emplace_back(new llama_mlock);
                 auto & mlock_buf = pimpl->mlock_bufs.back();
@@ -6106,10 +6107,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
                 buf_map.emplace(idx, buf);
             }
         }
-
-        if (pimpl->bufs.empty()) {
-            throw std::runtime_error("failed to allocate buffer");
-        }
+        pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), buf);
 
         for (auto & buf : buf_map) {
             // indicate that this buffer contains weights
@@ -6117,7 +6115,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
             ggml_backend_buffer_set_usage(buf.second, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
         }
 
-        ctx_bufs.emplace_back(ctx, buf_map);
+        ctx_buf_maps.emplace_back(ctx, buf_map);
     }
 
     if (llama_supports_gpu_offload()) {
@@ -6135,22 +6133,20 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
     }
 
     // print memory requirements per buffer type
-    for (auto & buf : pimpl->bufs) {
+    for (auto & [_, buf] : pimpl->ctxs_bufs) {
         LLAMA_LOG_INFO("%s: %12s model buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf.get()), ggml_backend_buffer_get_size(buf.get()) / 1024.0 / 1024.0);
     }
 
     // populate tensors_by_name
-    for (auto & ctx : pimpl->ctxs) {
+    for (auto & [ctx, _] : pimpl->ctxs_bufs) {
         for (auto * cur = ggml_get_first_tensor(ctx.get()); cur != NULL; cur = ggml_get_next_tensor(ctx.get(), cur)) {
             tensors_by_name.emplace_back(ggml_get_name(cur), cur);
         }
     }
 
     // load tensor data
-    for (auto & it : ctx_bufs) {
-        ggml_context * ctx = it.first;
-        auto & bufs = it.second;
-        if (!ml.load_all_data(ctx, bufs, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) {
+    for (auto & [ctx, buf_map] : ctx_buf_maps) {
+        if (!ml.load_all_data(ctx, buf_map, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) {
             return false;
         }
     }
@@ -6190,8 +6186,8 @@ size_t llama_model::n_devices() const {
 
 std::map<ggml_backend_buffer_type_t, size_t> llama_model::memory_breakdown() const {
     std::map<ggml_backend_buffer_type_t, size_t> ret;
-    for (const ggml_backend_buffer_ptr & buf_ptr : pimpl->bufs) {
-        ret[ggml_backend_buffer_get_type(buf_ptr.get())] += ggml_backend_buffer_get_size(buf_ptr.get());
+    for (const auto & [_, buf] : pimpl->ctxs_bufs) {
+        ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());
     }
     return ret;
 }