@fugood/llama.node 1.3.0-rc.0 → 1.3.0-rc.1

package/CMakeLists.txt

@@ -124,6 +124,8 @@ include_directories(
   ${CMAKE_JS_INC}
   "src/llama.cpp"
   "src/llama.cpp/src"
+  "src/llama.cpp/ggml/include"
+  "src/llama.cpp/ggml/src"
   "src/tools/mtmd"
 )
 

package/lib/binding.js

@@ -15,23 +15,13 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {

package/lib/binding.ts

@@ -236,6 +236,36 @@ export type LlamaCompletionToken = {
   completion_probabilities?: CompletionProbability[]
 }
 
+/**
+ * Result from a parallel completion request (queueCompletion callback).
+ * Extends the basic completion result with per-slot timing information.
+ */
+export type LlamaParallelCompletionResult = {
+  requestId: number
+  text: string
+  reasoning_content?: string
+  content?: string
+  tool_calls?: ToolCall[]
+  chat_format: number
+  stopped_eos: boolean
+  stopped_limit: boolean
+  stopped_word: boolean
+  context_full: boolean
+  tokens_evaluated: number
+  tokens_predicted: number
+  timings: {
+    cache_n: number
+    prompt_n: number
+    prompt_ms: number
+    prompt_per_token_ms: number
+    prompt_per_second: number
+    predicted_n: number
+    predicted_ms: number
+    predicted_per_token_ms: number
+    predicted_per_second: number
+  }
+}
+
 export type TokenizeResult = {
   tokens: Int32Array
   has_media: boolean
@@ -257,6 +287,14 @@ export type RerankResult = {
   index: number
 }
 
+export type BackendDeviceInfo = {
+  backend: string
+  type: string
+  deviceName: string
+  maxMemorySize: number
+  metadata?: Record<string, any>
+}
+
 export type ModelInfo = {
   desc: string
   nEmbd: number
@@ -457,12 +495,12 @@ export interface LlamaContext {
   /**
    * Queue a completion request for parallel processing
    * @param options Completion options with parallel-specific state management
-   * @param callback Optional token callback
+   * @param callback Optional callback that receives tokens during generation and final result
    * @returns Object with requestId
    */
   queueCompletion(
     options: LlamaParallelCompletionOptions,
-    callback?: (error: any, result: any) => void,
+    callback?: (error: any, result: LlamaParallelCompletionResult) => void,
   ): { requestId: number }
 
   /**
@@ -505,6 +543,11 @@ export interface LlamaContext {
     enable: boolean,
     callback: (level: string, text: string) => void,
   ): void
+  /**
+   * Get information about available backend devices
+   * @returns Array of backend device information
+   */
+  getBackendDevicesInfo(): BackendDeviceInfo[]
 }
 
 export interface Module {

package/lib/index.js

@@ -23,7 +23,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BuildInfo = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.MTMD_DEFAULT_MEDIA_MARKER = exports.LlamaParallelAPI = void 0;
+exports.BuildInfo = exports.getBackendDevicesInfo = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.MTMD_DEFAULT_MEDIA_MARKER = exports.LlamaParallelAPI = void 0;
 exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
 const version_1 = require("./version");
@@ -269,6 +269,14 @@ const loadLlamaModelInfo = (path) => __awaiter(void 0, void 0, void 0, function*
     return mods[variant].LlamaContext.loadModelInfo(path, modelInfoSkip);
 });
 exports.loadLlamaModelInfo = loadLlamaModelInfo;
+const getBackendDevicesInfo = (...args_1) => __awaiter(void 0, [...args_1], void 0, function* (variant = 'default') {
+    var _a;
+    (_a = mods[variant]) !== null && _a !== void 0 ? _a : (mods[variant] = yield (0, binding_1.loadModule)(variant));
+    refreshNativeLogSetup();
+    const jsonString = mods[variant].LlamaContext.getBackendDevicesInfo();
+    return JSON.parse(jsonString);
+});
+exports.getBackendDevicesInfo = getBackendDevicesInfo;
 exports.BuildInfo = {
     number: version_1.BUILD_NUMBER,
     commit: version_1.BUILD_COMMIT,

package/lib/index.ts

@@ -385,6 +385,15 @@ export const loadLlamaModelInfo = async (
   return mods[variant].LlamaContext.loadModelInfo(path, modelInfoSkip)
 }
 
+export const getBackendDevicesInfo = async (
+  variant: LibVariant = 'default'
+): Promise<import('./binding').BackendDeviceInfo[]> => {
+  mods[variant] ??= await loadModule(variant)
+  refreshNativeLogSetup()
+  const jsonString = mods[variant].LlamaContext.getBackendDevicesInfo()
+  return JSON.parse(jsonString as any)
+}
+
 export const BuildInfo = {
   number: BUILD_NUMBER,
   commit: BUILD_COMMIT,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.3.0-rc.0",
+  "version": "1.3.0-rc.1",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,19 +72,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.3.0-rc.0",
-    "@fugood/node-llama-linux-x64-vulkan": "1.3.0-rc.0",
-    "@fugood/node-llama-linux-x64-cuda": "1.3.0-rc.0",
-    "@fugood/node-llama-linux-arm64": "1.3.0-rc.0",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.3.0-rc.0",
-    "@fugood/node-llama-linux-arm64-cuda": "1.3.0-rc.0",
-    "@fugood/node-llama-win32-x64": "1.3.0-rc.0",
-    "@fugood/node-llama-win32-x64-vulkan": "1.3.0-rc.0",
-    "@fugood/node-llama-win32-x64-cuda": "1.3.0-rc.0",
-    "@fugood/node-llama-win32-arm64": "1.3.0-rc.0",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.3.0-rc.0",
-    "@fugood/node-llama-darwin-x64": "1.3.0-rc.0",
-    "@fugood/node-llama-darwin-arm64": "1.3.0-rc.0"
+    "@fugood/node-llama-linux-x64": "1.3.0-rc.1",
+    "@fugood/node-llama-linux-x64-vulkan": "1.3.0-rc.1",
+    "@fugood/node-llama-linux-x64-cuda": "1.3.0-rc.1",
+    "@fugood/node-llama-linux-arm64": "1.3.0-rc.1",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.3.0-rc.1",
+    "@fugood/node-llama-linux-arm64-cuda": "1.3.0-rc.1",
+    "@fugood/node-llama-win32-x64": "1.3.0-rc.1",
+    "@fugood/node-llama-win32-x64-vulkan": "1.3.0-rc.1",
+    "@fugood/node-llama-win32-x64-cuda": "1.3.0-rc.1",
+    "@fugood/node-llama-win32-arm64": "1.3.0-rc.1",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.3.0-rc.1",
+    "@fugood/node-llama-darwin-x64": "1.3.0-rc.1",
+    "@fugood/node-llama-darwin-arm64": "1.3.0-rc.1"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/src/LlamaContext.cpp

@@ -89,6 +89,13 @@ Napi::Value LlamaContext::ModelInfo(const Napi::CallbackInfo &info) {
   return metadata;
 }
 
+// getBackendDevicesInfo(): string
+Napi::Value LlamaContext::GetBackendDevicesInfo(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  std::string devices_json = rnllama::get_backend_devices_info();
+  return Napi::String::New(env, devices_json);
+}
+
 void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
   Napi::Function func = DefineClass(
       env, "LlamaContext",
@@ -148,6 +155,9 @@ void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
        StaticMethod<&LlamaContext::ToggleNativeLog>(
            "toggleNativeLog",
            static_cast<napi_property_attributes>(napi_enumerable)),
+       StaticMethod<&LlamaContext::GetBackendDevicesInfo>(
+           "getBackendDevicesInfo",
+           static_cast<napi_property_attributes>(napi_enumerable)),
        InstanceMethod<&LlamaContext::GetMultimodalSupport>(
            "getMultimodalSupport",
            static_cast<napi_property_attributes>(napi_enumerable)),

package/src/LlamaContext.h

@@ -25,6 +25,7 @@ public:
   ~LlamaContext();
   static void ToggleNativeLog(const Napi::CallbackInfo &info);
   static Napi::Value ModelInfo(const Napi::CallbackInfo &info);
+  static Napi::Value GetBackendDevicesInfo(const Napi::CallbackInfo &info);
   static void Init(Napi::Env env, Napi::Object &exports);
 
 private: