@fugood/llama.node 0.4.6 → 0.5.0

package/lib/binding.ts

@@ -6,6 +6,11 @@ export type MessagePart = {
   text?: string,
   image_url?: {
     url?: string
+  },
+  input_audio?: {
+    format: string
+    data?: string
+    url?: string
   }
 }
 
@@ -103,12 +108,12 @@ export type LlamaCompletionOptions = {
   grammar_triggers?: { type: number; word: string; at_start: boolean }[]
   preserved_tokens?: string[]
   /**
-   * Path(s) to image file(s) to process before generating text.
-   * When provided, the image(s) will be processed and added to the context.
+   * Path(s) to media file(s) to process before generating text.
+   * When provided, the media will be processed and added to the context.
    * Requires multimodal support to be enabled via initMultimodal.
    * Supports both file paths and base64 data URLs.
    */
-  image_paths?: string | string[]
+  media_paths?: string | string[]
 }
 
 export type LlamaCompletionResult = {
@@ -137,6 +142,10 @@ export type LlamaCompletionToken = {
 
 export type TokenizeResult = {
   tokens: Int32Array
+  has_media: boolean
+  bitmap_hashes: string[]
+  chunk_pos: number[]
+  chunk_pos_media: number[]
 }
 
 export type EmbeddingResult = {
@@ -163,7 +172,7 @@ export interface LlamaContext {
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult>
   stopCompletion(): void
-  tokenize(text: string): Promise<TokenizeResult>
+  tokenize(text: string, media_paths?: string[]): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
   embedding(text: string): Promise<EmbeddingResult>
   saveSession(path: string): Promise<void>
@@ -185,6 +194,15 @@ export interface LlamaContext {
    */
   isMultimodalEnabled(): Promise<boolean>
 
+  /**
+   * Get multimodal support capabilities
+   * @returns Promise resolving to an object with vision and audio support
+   */
+  getMultimodalSupport(): Promise<{
+    vision: boolean
+    audio: boolean
+  }>
+
   /**
    * Release multimodal support
    */

package/lib/index.js

@@ -23,10 +23,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = void 0;
+exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.MTMD_DEFAULT_MEDIA_MARKER = void 0;
 exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
 __exportStar(require("./binding"), exports);
+exports.MTMD_DEFAULT_MEDIA_MARKER = '<__media__>';
 const mods = {};
 const logListeners = [];
 const logCallback = (level, text) => {
@@ -78,13 +79,13 @@ class LlamaContextWrapper {
     isLlamaChatSupported() {
         return !!this.ctx.getModelInfo().chatTemplates.llamaChat;
     }
-    _formatImageChat(messages) {
+    _formatMediaChat(messages) {
         if (!messages)
             return {
                 messages,
-                has_image: false,
+                has_media: false,
             };
-        const imagePaths = [];
+        const mediaPaths = [];
         return {
             messages: messages.map((msg) => {
                 if (Array.isArray(msg.content)) {
@@ -93,10 +94,30 @@ class LlamaContextWrapper {
                         // Handle multimodal content
                         if (part.type === 'image_url') {
                             let path = ((_a = part.image_url) === null || _a === void 0 ? void 0 : _a.url) || '';
-                            imagePaths.push(path);
+                            mediaPaths.push(path);
                             return {
                                 type: 'text',
-                                text: '<__image__>',
+                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
+                            };
+                        }
+                        else if (part.type === 'input_audio') {
+                            const { input_audio: audio } = part;
+                            if (!audio)
+                                throw new Error('input_audio is required');
+                            const { format } = audio;
+                            if (format != 'wav' && format != 'mp3') {
+                                throw new Error(`Unsupported audio format: ${format}`);
+                            }
+                            if (audio.url) {
+                                const path = audio.url.replace(/file:\/\//, '');
+                                mediaPaths.push(path);
+                            }
+                            else if (audio.data) {
+                                mediaPaths.push(audio.data);
+                            }
+                            return {
+                                type: 'text',
+                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
                             };
                         }
                         return part;
@@ -105,14 +126,14 @@ class LlamaContextWrapper {
                 }
                 return msg;
             }),
-            has_image: imagePaths.length > 0,
-            image_paths: imagePaths,
+            has_media: mediaPaths.length > 0,
+            media_paths: mediaPaths,
         };
     }
     getFormattedChat(messages, template, params) {
-        const { messages: chat, has_image, image_paths, } = this._formatImageChat(messages);
+        const { messages: chat, has_media, media_paths, } = this._formatMediaChat(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
-        let tmpl = this.isLlamaChatSupported() || useJinja ? undefined : 'chatml';
+        let tmpl;
         if (template)
             tmpl = template; // Force replace if provided
         const jsonSchema = getJsonSchema(params === null || params === void 0 ? void 0 : params.response_format);
@@ -127,25 +148,25 @@ class LlamaContextWrapper {
             return {
                 type: 'llama-chat',
                 prompt: result,
-                has_image,
-                image_paths,
+                has_media,
+                media_paths,
             };
         }
         const jinjaResult = result;
         jinjaResult.type = 'jinja';
-        jinjaResult.has_image = has_image;
-        jinjaResult.image_paths = image_paths;
+        jinjaResult.has_media = has_media;
+        jinjaResult.media_paths = media_paths;
         return jinjaResult;
     }
     completion(options, callback) {
-        const { messages, image_paths = options.image_paths } = this._formatImageChat(options.messages);
-        return this.ctx.completion(Object.assign(Object.assign({}, options), { messages, image_paths: options.image_paths || image_paths }), callback || (() => { }));
+        const { messages, media_paths = options.media_paths } = this._formatMediaChat(options.messages);
+        return this.ctx.completion(Object.assign(Object.assign({}, options), { messages, media_paths: options.media_paths || media_paths }), callback || (() => { }));
     }
     stopCompletion() {
         return this.ctx.stopCompletion();
     }
-    tokenize(text) {
-        return this.ctx.tokenize(text);
+    tokenize(text, { media_paths } = {}) {
+        return this.ctx.tokenize(text, media_paths);
     }
     detokenize(tokens) {
         return this.ctx.detokenize(tokens);
@@ -180,6 +201,9 @@ class LlamaContextWrapper {
     releaseMultimodal() {
         return this.ctx.releaseMultimodal();
     }
+    getMultimodalSupport() {
+        return this.ctx.getMultimodalSupport();
+    }
 }
 const loadModel = (options) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;

package/lib/index.ts

@@ -14,6 +14,8 @@ import type {
 
 export * from './binding'
 
+export const MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'
+
 export interface LlamaModelOptionsExtended extends LlamaModelOptions {
   lib_variant?: LibVariant
 }
@@ -60,6 +62,13 @@ const getJsonSchema = (responseFormat?: CompletionResponseFormat) => {
   return null
 }
 
+export type FormattedChatResult = {
+  type: 'jinja' | 'llama-chat'
+  prompt: string
+  has_media: boolean
+  media_paths?: Array<string>
+}
+
 class LlamaContextWrapper {
   ctx: any
 
@@ -84,17 +93,17 @@ class LlamaContextWrapper {
     return !!this.ctx.getModelInfo().chatTemplates.llamaChat
   }
 
-  _formatImageChat(messages: ChatMessage[] | undefined): {
+  _formatMediaChat(messages: ChatMessage[] | undefined): {
     messages: ChatMessage[] | undefined
-    has_image: boolean
-    image_paths?: string[]
+    has_media: boolean
+    media_paths?: string[]
   } {
     if (!messages)
       return {
         messages,
-        has_image: false,
+        has_media: false,
       }
-    const imagePaths: string[] = []
+    const mediaPaths: string[] = []
     return {
       messages: messages.map((msg) => {
         if (Array.isArray(msg.content)) {
@@ -102,10 +111,28 @@ class LlamaContextWrapper {
             // Handle multimodal content
             if (part.type === 'image_url') {
               let path = part.image_url?.url || ''
-              imagePaths.push(path)
+              mediaPaths.push(path)
               return {
                 type: 'text',
-                text: '<__image__>',
+                text: MTMD_DEFAULT_MEDIA_MARKER,
+              }
+            } else if (part.type === 'input_audio') {
+              const { input_audio: audio } = part
+              if (!audio) throw new Error('input_audio is required')
+
+              const { format } = audio
+              if (format != 'wav' && format != 'mp3') {
+                throw new Error(`Unsupported audio format: ${format}`)
+              }
+              if (audio.url) {
+                const path = audio.url.replace(/file:\/\//, '')
+                mediaPaths.push(path)
+              } else if (audio.data) {
+                mediaPaths.push(audio.data)
+              }
+              return {
+                type: 'text',
+                text: MTMD_DEFAULT_MEDIA_MARKER,
               }
             }
             return part
@@ -118,8 +145,8 @@ class LlamaContextWrapper {
         }
         return msg
       }),
-      has_image: imagePaths.length > 0,
-      image_paths: imagePaths,
+      has_media: mediaPaths.length > 0,
+      media_paths: mediaPaths,
     }
   }
 
@@ -133,15 +160,15 @@ class LlamaContextWrapper {
       parallel_tool_calls?: object
       tool_choice?: string
     },
-  ): object {
+  ): FormattedChatResult {
     const {
       messages: chat,
-      has_image,
-      image_paths,
-    } = this._formatImageChat(messages)
+      has_media,
+      media_paths,
+    } = this._formatMediaChat(messages)
 
     const useJinja = this.isJinjaSupported() && params?.jinja
-    let tmpl = this.isLlamaChatSupported() || useJinja ? undefined : 'chatml'
+    let tmpl
     if (template) tmpl = template // Force replace if provided
     const jsonSchema = getJsonSchema(params?.response_format)
 
@@ -157,14 +184,14 @@ class LlamaContextWrapper {
       return {
         type: 'llama-chat',
         prompt: result as string,
-        has_image,
-        image_paths,
+        has_media,
+        media_paths,
       }
     }
     const jinjaResult = result
     jinjaResult.type = 'jinja'
-    jinjaResult.has_image = has_image
-    jinjaResult.image_paths = image_paths
+    jinjaResult.has_media = has_media
+    jinjaResult.media_paths = media_paths
     return jinjaResult
   }
 
@@ -172,12 +199,12 @@ class LlamaContextWrapper {
     options: LlamaCompletionOptions,
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult> {
-    const { messages, image_paths = options.image_paths } =
-      this._formatImageChat(options.messages)
+    const { messages, media_paths = options.media_paths } =
+      this._formatMediaChat(options.messages)
     return this.ctx.completion({
       ...options,
       messages,
-      image_paths: options.image_paths || image_paths,
+      media_paths: options.media_paths || media_paths,
     }, callback || (() => {}))
   }
 
@@ -185,8 +212,8 @@ class LlamaContextWrapper {
     return this.ctx.stopCompletion()
   }
 
-  tokenize(text: string): Promise<TokenizeResult> {
-    return this.ctx.tokenize(text)
+  tokenize(text: string, { media_paths }: { media_paths?: string[] } = {}): Promise<TokenizeResult> {
+    return this.ctx.tokenize(text, media_paths)
   }
 
   detokenize(tokens: number[]): Promise<string> {
@@ -235,6 +262,13 @@ class LlamaContextWrapper {
   releaseMultimodal(): Promise<void> {
     return this.ctx.releaseMultimodal()
   }
+
+  getMultimodalSupport(): Promise<{
+    vision: boolean
+    audio: boolean
+  }> {
+    return this.ctx.getMultimodalSupport()
+  }
 }
 
 export const loadModel = async (

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.4.6",
+  "version": "0.5.0",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {