@fugood/bricks-project 2.22.0-beta.10 → 2.22.0-beta.12

package/compile/action-name-map.ts

@@ -552,6 +552,8 @@ export const templateActionNameMap = {
       prompt: 'GENERATOR_ONNX_LLM_PROMPT',
       chat: 'GENERATOR_ONNX_LLM_CHAT',
       images: 'GENERATOR_ONNX_LLM_IMAGES',
+      tools: 'GENERATOR_ONNX_LLM_TOOLS',
+      toolChoice: 'GENERATOR_ONNX_LLM_TOOL_CHOICE',
     },
   },
   GENERATOR_ONNX_STT: {
@@ -590,6 +592,14 @@ export const templateActionNameMap = {
     },
   },
   GENERATOR_LLM: {
+    GENERATOR_LLM_TOKENIZE: {
+      mode: 'GENERATOR_LLM_MODE',
+      messages: 'GENERATOR_LLM_MESSAGES',
+      prompt: 'GENERATOR_LLM_PROMPT',
+    },
+    GENERATOR_LLM_DETOKENIZE: {
+      tokens: 'GENERATOR_LLM_TOKENS',
+    },
     GENERATOR_LLM_PROCESS_PROMPT: {
       sessionKey: 'GENERATOR_LLM_SESSION_KEY',
       mode: 'GENERATOR_LLM_MODE',
@@ -598,6 +608,7 @@ export const templateActionNameMap = {
       parallelToolCalls: 'GENERATOR_LLM_PARALLEL_TOOL_CALLS',
       toolChoice: 'GENERATOR_LLM_TOOL_CHOICE',
       prompt: 'GENERATOR_LLM_PROMPT',
+      promptImagePaths: 'GENERATOR_LLM_PROMPT_IMAGE_PATHS',
       promptTemplateData: 'GENERATOR_LLM_PROMPT_TEMPLATE_DATA',
       promptTemplateType: 'GENERATOR_LLM_PROMPT_TEMPLATE_TYPE',
       responseFormat: 'GENERATOR_LLM_RESPONSE_FORMAT',
@@ -610,6 +621,7 @@ export const templateActionNameMap = {
       parallelToolCalls: 'GENERATOR_LLM_PARALLEL_TOOL_CALLS',
       toolChoice: 'GENERATOR_LLM_TOOL_CHOICE',
       prompt: 'GENERATOR_LLM_PROMPT',
+      promptImagePaths: 'GENERATOR_LLM_PROMPT_IMAGE_PATHS',
       promptTemplateData: 'GENERATOR_LLM_PROMPT_TEMPLATE_DATA',
       promptTemplateType: 'GENERATOR_LLM_PROMPT_TEMPLATE_TYPE',
       responseFormat: 'GENERATOR_LLM_RESPONSE_FORMAT',
@@ -650,6 +662,7 @@ export const templateActionNameMap = {
     GENERATOR_QNN_LLM_GENERATE: {
       prompt: 'GENERATOR_QNN_LLM_PROMPT',
       messages: 'GENERATOR_QNN_LLM_MESSAGES',
+      tools: 'GENERATOR_QNN_LLM_TOOLS',
     },
   },
   GENERATOR_OPENAI_LLM: {
@@ -676,6 +689,7 @@ export const templateActionNameMap = {
     GENERATOR_ASSISTANT_ADD_MESSAGE: {
       role: 'GENERATOR_ASSISTANT_ROLE',
       content: 'GENERATOR_ASSISTANT_CONTENT',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
       useFileSearch: 'GENERATOR_ASSISTANT_USE_FILE_SEARCH',
       filePath: 'GENERATOR_ASSISTANT_FILE_PATH',
@@ -699,12 +713,14 @@ export const templateActionNameMap = {
     GENERATOR_ASSISTANT_UPDATE_MESSAGE_AT_INDEX: {
       index: 'GENERATOR_ASSISTANT_INDEX',
       content: 'GENERATOR_ASSISTANT_CONTENT',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
     },
     GENERATOR_ASSISTANT_ADD_AUDIO_MESSAGE: {
       role: 'GENERATOR_ASSISTANT_ROLE',
       contentFile: 'GENERATOR_ASSISTANT_CONTENT_FILE',
       contentBase64: 'GENERATOR_ASSISTANT_CONTENT_BASE64',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       useFileSearch: 'GENERATOR_ASSISTANT_USE_FILE_SEARCH',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
       filePath: 'GENERATOR_ASSISTANT_FILE_PATH',
@@ -727,6 +743,7 @@ export const templateActionNameMap = {
       index: 'GENERATOR_ASSISTANT_INDEX',
       contentFile: 'GENERATOR_ASSISTANT_CONTENT_FILE',
       contentBase64: 'GENERATOR_ASSISTANT_CONTENT_BASE64',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
     },
     GENERATOR_ASSISTANT_REMOVE_MESSAGE_AT_INDEX: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fugood/bricks-project",
-  "version": "2.22.0-beta.10",
+  "version": "2.22.0-beta.12",
   "main": "index.ts",
   "scripts": {
     "build": "node scripts/build.js"
@@ -14,5 +14,5 @@
     "lodash": "^4.17.4",
     "uuid": "^8.3.1"
   },
-  "gitHead": "a66b1a510bfd78709dc6251ef5b83bc470049308"
+  "gitHead": "bf844dee8ddb0e8437c8795e0a44cafc48ef16d1"
 }

package/types/bricks.ts

@@ -1870,7 +1870,7 @@ Default property:
     verticalMaxQuantity?: number | DataLink
     /* Resize mode */
     resizeMode?: 'auto' | 'fix' | DataLink
-    /* Align Content */
+    /* Justify Content */
     justifyContent?: 'start' | 'end' | 'center' | 'space-between' | 'space-around' | DataLink
     /* Align Content */
     alignContent?:

package/types/generators.ts

@@ -47,7 +47,7 @@ Default property:
     completed?: Array<EventAction>
   }
   outlets?: {
-    /* Result of each countdown change */
+    /* Countdown step value */
     countdown?: () => Data
   }
 }
@@ -691,7 +691,7 @@ Default property:
 }
   */
   property?: {
-    /* Start query on generator initialized */
+    /* Start generator initialization execution immediately */
     init?: boolean | DataLink
     /* Data Bank Space ID */
     spacename?: string | DataLink
@@ -730,7 +730,7 @@ Default property:
   }
 }
 
-/* Fetch data & subscribe data change from Data Bank */
+/* Get Data or subscribe to Data changes from Data Bank */
 export type GeneratorDataBank = Generator &
   GeneratorDataBankDef & {
     templateKey: 'GENERATOR_DATA_BANK'
@@ -752,7 +752,7 @@ export type GeneratorDataBank = Generator &
     >
   }
 
-/* Run GraphQL query with defined properties */
+/* Execute GraphQL request with defined properties */
 export type GeneratorGraphQLActionRunQuery = ActionWithParams & {
   __actionName: 'GENERATOR_GRAPHQL_RUN_QUERY'
   params?: Array<
@@ -797,7 +797,7 @@ Default property:
 }
   */
   property?: {
-    /* Start GraphQL query on generator initialized */
+    /* Start GraphQL request immediately after generator initialization */
     init?: boolean | DataLink
     /* GraphQL request type */
     type?: 'query' | 'mutation' | 'subscription' | DataLink
@@ -805,9 +805,9 @@ Default property:
     headers?: {} | DataLink
     /* HTTP request URL endpoint */
     endpoint?: string | DataLink
-    /* Subscrpition endpoint */
+    /* Subscription endpoint */
     endpointForSubscription?: string | DataLink
-    /* Subscrpition connection params */
+    /* Subscription connection params */
     connectionParams?: {} | DataLink
     /* Query content */
     query?: string | DataLink
@@ -831,9 +831,9 @@ Default property:
     firebaseMessagingSenderId?: string | DataLink
   }
   events?: {
-    /* Event of subscription on connection */
+    /* Event triggered when subscription connection is successful */
     subscriptionOnConnection?: Array<EventAction>
-    /* Server connections error of GraphQL subscription */
+    /* Event triggered when subscription connection error occurs */
     subscriptionOnConnectionError?: Array<EventAction>
   }
   outlets?: {
@@ -973,22 +973,22 @@ export type GeneratorHTTP = Generator &
     >
   }
 
-/* Start play sound */
+/* Start playing sound */
 export type GeneratorSoundPlayerActionPlay = Action & {
   __actionName: 'GENERATOR_SOUND_PLAYER_PLAY'
 }
 
-/* Pause play sound */
+/* Pause playing sound */
 export type GeneratorSoundPlayerActionPause = Action & {
   __actionName: 'GENERATOR_SOUND_PLAYER_PAUSE'
 }
 
-/* Resume play sound from pause */
+/* Resume playing sound from pause */
 export type GeneratorSoundPlayerActionResume = Action & {
   __actionName: 'GENERATOR_SOUND_PLAYER_RESUME'
 }
 
-/* Stop play sound */
+/* Stop playing sound */
 export type GeneratorSoundPlayerActionRelease = Action & {
   __actionName: 'GENERATOR_SOUND_PLAYER_RELEASE'
 }
@@ -1002,32 +1002,32 @@ Default property:
 }
   */
   property?: {
-    /* The sound file path */
+    /* Sound file path */
     filePath?: string | DataLink
-    /* The checksum of file */
+    /* MD5 */
     md5?: string | DataLink
-    /* Repeat the sound */
+    /* Repeat playback */
     loop?: boolean | DataLink
-    /* The volume of sound (0 - 100) */
+    /* Sound volume (0 - 100) */
     volume?: number | DataLink
   }
   events?: {
-    /* Event on sound file loaded */
+    /* Sound file loaded successfully */
     onLoad?: Array<EventAction>
-    /* Event on load error */
+    /* Sound file load error */
     onLoadError?: Array<EventAction>
-    /* Event on sound play */
+    /* Sound playback complete */
     onPlay?: Array<EventAction>
-    /* Event on sound end */
+    /* Sound file playback end */
     onEnd?: Array<EventAction>
   }
   outlets?: {
-    /* Is sound playing */
+    /* Whether the sound is playing */
     isPlaying?: () => Data
   }
 }
 
-/* Play sound file from file system,  support sound format refer this  https://developer.android.com/guide/topics/media/media-formats */
+/* Play sound, see supported formats at https://developer.android.com/guide/topics/media/media-formats */
 export type GeneratorSoundPlayer = Generator &
   GeneratorSoundPlayerDef & {
     templateKey: 'GENERATOR_SOUND_PLAYER'
@@ -1062,43 +1062,42 @@ Default property:
 }
   */
   property?: {
-    /* Enable listening */
+    /* Enable listening for input */
     enabled?: boolean | DataLink
-    /* Key map to convert key or key code to specify content (e.g. { 37: 'left' }) */
+    /* Key map to transform key or key code to the designated content (e.g. { 37: 'left' }) */
     keyMap?: {} | DataLink
-    /* Key outlet prefer use key code or key.  
-Please note that the key code is not supported on iOS / tvOS, so it will use `key` if value is `auto`. */
+    /* Key outlet preference use key code or key. */
     keyOutletPrefer?: 'auto' | 'key-code' | 'key' | DataLink
-    /* Stop key or code to finish batch */
+    /* Key or code to finish continuous input */
     batchStopKeys?: Array<string | DataLink | number | DataLink | DataLink> | DataLink
-    /* Debounce time to finish batch (ms) */
+    /* Debounce time (ms) to finish continuous input */
     batchDebounce?: number | DataLink
-    /* Max wait time to finish batch (ms) (Default: No limit) */
+    /* Maximum wait time (ms) to finish continuous input (default: unlimited) */
     batchDebounceMaxWait?: number | DataLink
   }
   events?: {
-    /* Event of key down */
+    /* Event on key press */
     onDown?: Array<EventAction>
-    /* Event of key up */
+    /* Event on key up */
     onUp?: Array<EventAction>
-    /* Event of batch input finished */
+    /* Event on continuous input complete */
     onBatch?: Array<EventAction>
   }
   outlets?: {
-    /* Last key down code */
+    /* Last key code pressed */
     lastKeyDown?: () => Data
-    /* Last key down flags */
+    /* Modifier key information on last key press */
     lastKeyDownFlags?: () => Data
-    /* Last key up code */
+    /* Last key code released */
     lastKeyUp?: () => Data
-    /* Last key up flags */
+    /* Modifier key information on last key release */
     lastKeyUpFlags?: () => Data
-    /* Last batch events */
+    /* Last continuous event */
     lastBatchEvents?: () => Data
   }
 }
 
-/* Listening keyboard (controller) event */
+/* Access keyboard (remote control) events */
 export type GeneratorKeyboard = Generator &
   GeneratorKeyboardDef & {
     templateKey: 'GENERATOR_KEYBOARD'
@@ -1395,22 +1394,22 @@ export type GeneratorStep = Generator &
     >
   }
 
-/* Start the next iterate */
+/* Proceed to next iteration */
 export type GeneratorIteratorActionNext = Action & {
   __actionName: 'GENERATOR_ITERATOR_NEXT'
 }
 
-/* Back to the previous iterate */
+/* Go back to previous iteration */
 export type GeneratorIteratorActionPrevious = Action & {
   __actionName: 'GENERATOR_ITERATOR_PREVIOUS'
 }
 
-/* Skip to last iterate element (Ignore loop) */
+/* Jump to the last iteration element (ignoring the loop setting) */
 export type GeneratorIteratorActionLast = Action & {
   __actionName: 'GENERATOR_ITERATOR_LAST'
 }
 
-/* Reset the iterator state */
+/* Reset iteration state */
 export type GeneratorIteratorActionReset = Action & {
   __actionName: 'GENERATOR_ITERATOR_RESET'
 }
@@ -1425,40 +1424,40 @@ Default property:
 }
   */
   property?: {
-    /* The data source of the value. If it is an array, the value element is used. If it is an object, Object.values() is used as the data. If it is a string, the source character of the value is taken. If it is a number, it represents a count of 1N. */
+    /* Data source for iteration. If it's an Array, it will iterate through elements. If it's an Object, it will use Object.values() as data source. If it's a String, it will iterate through characters. If it's a Number, it represents count from 1 to N. */
     data?: any
-    /* Starting element index */
+    /* Starting element position */
     start?: number | DataLink
-    /* Iterate step */
+    /* Step size for each iteration */
     step?: number | DataLink
-    /* The maximum number of iterations (Set -1 for unlimited) */
+    /* Maximum number of iterations (can be set to -1 for unlimited) */
     maxQuantity?: number | DataLink
-    /* Loop iterate */
+    /* Whether to loop the iteration */
     loop?: boolean | DataLink
   }
   events?: {
-    /* Event on iterate */
+    /* Event triggered on each iteration */
     iterate?: Array<EventAction>
-    /* Event on iterate round start */
+    /* Event triggered on the first iteration of a round */
     first?: Array<EventAction>
-    /* Event on iterate round end */
+    /* Event triggered on the last iteration of a round */
     end?: Array<EventAction>
   }
   outlets?: {
-    /* Elements that have been iterated (Included current iterated) */
+    /* Elements that have been iterated (including current one) */
     iteratedArray?: () => Data
-    /* Elements not yet iterated */
+    /* Elements that will be iterated but have not been iterated yet */
     upcomingArray?: () => Data
-    /* Current iterated element */
+    /* Current iteration element */
     value?: () => Data
-    /* Current Key of iterated element (number: same as value; array, string: index; object: string) */
+    /* Key of the current iteration element (for number: same as value, for array/string: index, for object: string key) */
     key?: () => Data
-    /* The current number of iterations (if data is 6 and step is 2, this value will be returned in order: 1, 2, 3) */
+    /* Current iteration count (if data is 6 and step is 2, this will return: 1,2,3 in sequence) */
     index?: () => Data
   }
 }
 
-/* Iterate values (Array, Object, Number, String) */
+/* Iterate through values (Array, Object, Number, String) */
 export type GeneratorIterator = Generator &
   GeneratorIteratorDef & {
     templateKey: 'GENERATOR_ITERATOR'
@@ -2065,14 +2064,11 @@ Default property:
 }
   */
   property?: {
-    /* Try attach on generator initialized  
-  On web require user activation to attach device */
+    /* Try attach on generator initialized  On web require user activation to attach device */
     attachOnInit?: boolean | DataLink
     /* The serial device driver */
     driver?: 'fd' | 'usb' | DataLink
-    /* The serial device path  
-  e.g. /dev/ttyS0 or /dev/bus/usb/001/001  
-  For desktop and web is device index number */
+    /* The serial device path  e.g. /dev/ttyS0 or /dev/bus/usb/001/001  For desktop and web is device index number */
     path?: string | DataLink
     /* The serial USB vendor id (autoconnect first) */
     vendorId?: number | DataLink
@@ -4349,7 +4345,9 @@ Default property:
     `detect-data-change`: Watch data target change to return data, 
       please update data with ({ id: string, content: string | object }), 
       and ensure the id is same with request id  
-    `script`: Not implemented yet */
+    `script`: Run a JavaScript code to return data  
+      - Script can define members to call generator functions  
+      - Script support async/await */
     resources?:
       | Array<
           | DataLink
@@ -4365,7 +4363,7 @@ Default property:
                 | {
                     target?: string | DataLink
                     timeout?: number | DataLink
-                    payload?: any
+                    additionalParams?: {} | DataLink
                   }
               scriptConfig?:
                 | DataLink
@@ -4381,7 +4379,7 @@ Default property:
                             }
                         >
                       | DataLink
-                    payload?: any
+                    additionalParams?: {} | DataLink
                   }
             }
         >
@@ -4391,7 +4389,9 @@ Default property:
     `detect-data-change`: Watch data target change to return data, 
       please update data with ({ id: string, content: string | object }), 
       and ensure the id is same with request id.  
-    `script`: Not implemented yet */
+    `script`: Run a JavaScript code to return data  
+      - Script can define members to call generator functions  
+      - Script support async/await */
     tools?:
       | Array<
           | DataLink
@@ -4406,7 +4406,7 @@ Default property:
                 | {
                     target?: string | DataLink
                     timeout?: number | DataLink
-                    payload?: any
+                    additionalParams?: {} | DataLink
                   }
               scriptConfig?:
                 | DataLink
@@ -4422,7 +4422,7 @@ Default property:
                             }
                         >
                       | DataLink
-                    payload?: any
+                    additionalParams?: {} | DataLink
                   }
             }
         >
@@ -4433,7 +4433,9 @@ Default property:
     `detect-data-change`: Watch data target change to return data, 
       please update data with ({ id: string, content: string | object }), 
       and ensure the id is same with request id  
-    `script`: Not implemented yet */
+    `script`: Run a JavaScript code to return data  
+      - Script can define members to call generator functions  
+      - Script support async/await */
     prompts?:
       | Array<
           | DataLink
@@ -4449,7 +4451,7 @@ Default property:
                 | {
                     target?: string | DataLink
                     timeout?: number | DataLink
-                    payload?: any
+                    additionalParams?: {} | DataLink
                   }
               scriptConfig?:
                 | DataLink
@@ -4465,7 +4467,7 @@ Default property:
                             }
                         >
                       | DataLink
-                    payload?: any
+                    additionalParams?: {} | DataLink
                   }
             }
         >
@@ -4998,6 +5000,16 @@ export type GeneratorOnnxLLMActionInfer = ActionWithParams & {
         value?: Array<any> | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'tools'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'toolChoice'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
   >
 }
 
@@ -5017,6 +5029,8 @@ Default property:
 {
   "model": "BricksDisplay/phi-1_5-q4",
   "modelType": "auto",
+  "toolCallParser": "llama3_json",
+  "toolChoice": "auto",
   "maxNewTokens": 256,
   "temperature": 0.7,
   "topK": 50,
@@ -5123,6 +5137,14 @@ Default property:
     messages?: Array<DataLink | {}> | DataLink
     /* Images with message to inference */
     images?: Array<string | DataLink> | DataLink
+    /* Tool call parser */
+    toolCallParser?: 'llama3_json' | 'mistral' | 'hermes' | 'internlm' | 'phi4' | DataLink
+    /* Tools for chat mode using OpenAI-compatible function calling format  
+ Format: Array of objects with {type, function: {name, description, parameters}} structure  
+ See: https://platform.openai.com/docs/guides/function-calling */
+    tools?: Array<{} | DataLink> | DataLink
+    /* Tool choice for chat mode */
+    toolChoice?: 'none' | 'auto' | DataLink
     /* Max new tokens to generate */
     maxNewTokens?: number | DataLink
     /* Temperature */
@@ -5157,6 +5179,10 @@ Default property:
   events?: {
     /* Event triggered when state change */
     onContextStateChange?: Array<EventAction>
+    /* Event triggered on get function call request */
+    onFunctionCall?: Array<EventAction>
+    /* Event triggered on completion finished */
+    onCompletionFinished?: Array<EventAction>
     /* Event triggered when error occurs */
     onError?: Array<EventAction>
   }
@@ -5167,6 +5193,8 @@ Default property:
     generated?: () => Data
     /* Full result of generation */
     fullResult?: () => Data
+    /* Last function call */
+    lastFunctionCall?: () => Data
   }
 }
 
@@ -5185,7 +5213,7 @@ export type GeneratorOnnxLLM = Generator &
               | SwitchCondData
               | {
                   __typename: 'SwitchCondInnerStateOutlet'
-                  outlet: 'contextState' | 'generated' | 'fullResult'
+                  outlet: 'contextState' | 'generated' | 'fullResult' | 'lastFunctionCall'
                   value: any
                 }
           }>
@@ -5876,6 +5904,43 @@ export type GeneratorLLMActionLoadModel = Action & {
   __actionName: 'GENERATOR_LLM_LOAD_MODEL'
 }
 
+/* Load multimodal (vision) model (PREVIEW FEATURE) */
+export type GeneratorLLMActionLoadMultimodalModel = Action & {
+  __actionName: 'GENERATOR_LLM_LOAD_MULTIMODAL_MODEL'
+}
+
+/* Tokenize the prompt */
+export type GeneratorLLMActionTokenize = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_TOKENIZE'
+  params?: Array<
+    | {
+        input: 'mode'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'messages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Detokenize the tokens to text */
+export type GeneratorLLMActionDetokenize = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_DETOKENIZE'
+  params?: Array<{
+    input: 'tokens'
+    value?: Array<any> | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
 /* Pre-process the prompt, this can speed up the completion action */
 export type GeneratorLLMActionProcessPrompt = ActionWithParams & {
   __actionName: 'GENERATOR_LLM_PROCESS_PROMPT'
@@ -5915,6 +5980,11 @@ export type GeneratorLLMActionProcessPrompt = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'promptImagePaths'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'promptTemplateData'
         value?: {} | DataLink | EventProperty
@@ -5972,6 +6042,11 @@ export type GeneratorLLMActionCompletion = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'promptImagePaths'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'promptTemplateData'
         value?: {} | DataLink | EventProperty
@@ -6152,6 +6227,11 @@ export type GeneratorLLMActionClearDownload = Action & {
   __actionName: 'GENERATOR_LLM_CLEAR_DOWNLOAD'
 }
 
+/* Release multimodal (vision) context (PREVIEW FEATURE) */
+export type GeneratorLLMActionReleaseMultimodalContext = Action & {
+  __actionName: 'GENERATOR_LLM_RELEASE_MULTIMODAL_CONTEXT'
+}
+
 /* Release context */
 export type GeneratorLLMActionReleaseContext = Action & {
   __actionName: 'GENERATOR_LLM_RELEASE_CONTEXT'
@@ -6174,7 +6254,7 @@ Default property:
   "cacheVType": "f16",
   "ctxShift": true,
   "transformScriptEnabled": false,
-  "transformScriptCode": "\/\* Global variable: inputs = { prompt, messages, variables } \*\/\nreturn inputs.prompt",
+  "transformScriptCode": "\/\* Global variable: inputs = { prompt, messages, variables }, members = { llmUtils } \*\/\nreturn inputs.prompt",
   "transformScriptVariables": {},
   "sessionMinSaveSize": 50,
   "sessionRemain": 10,
@@ -6229,6 +6309,14 @@ Default property:
     modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
     /* Hash of model */
     modelHash?: string | DataLink
+    /* Load multimodal (vision) context after model loaded (PREVIEW FEATURE) */
+    initMultimodal?: boolean | DataLink
+    /* The URL or path of mmproj file for multimodal vision support (PREVIEW FEATURE) */
+    mmprojUrl?: string | DataLink
+    /* Hash type of mmproj file (PREVIEW FEATURE) */
+    mmprojHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of mmproj file (PREVIEW FEATURE) */
+    mmprojHash?: string | DataLink
     /* Chat Template (Jinja format) to override the default template from model */
     chatTemplate?: string | DataLink
     /* Context size (0 ~ 4096) (Default to 512) */
@@ -6298,6 +6386,9 @@ Default property:
       | DataLink
     /* Prompt (text mode) */
     completionPrompt?: string | DataLink
+    /* Image paths to be used in the prompt template (PREVIEW FEATURE)  
+ In prompt, use `<__image__>` for position of image content */
+    completionPromptImagePaths?: Array<string | DataLink> | DataLink
     /* Data to be used in the prompt template (e.g. `Hello ${name}`). Supports nested data, such as `Hello ${user.name}`. */
     completionPromptTemplateData?: {} | DataLink
     /* The prompt template type */
@@ -6395,6 +6486,10 @@ Default property:
     sessions?: () => Data
     /* Is evaluating */
     isEvaluating?: () => Data
+    /* Tokenize result */
+    tokenizeResult?: () => Data
+    /* Detokenize result */
+    detokenizeResult?: () => Data
     /* Last formatted prompt (messages or prompt) */
     completionLastFormattedPrompt?: () => Data
     /* Last completion token */
@@ -6435,6 +6530,8 @@ export type GeneratorLLM = Generator &
                     | 'contextDetails'
                     | 'sessions'
                     | 'isEvaluating'
+                    | 'tokenizeResult'
+                    | 'detokenizeResult'
                     | 'completionLastFormattedPrompt'
                     | 'completionLastToken'
                     | 'completionResult'
@@ -6471,6 +6568,11 @@ export type GeneratorQnnLlmActionGenerate = ActionWithParams & {
         value?: Array<any> | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'tools'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
   >
 }
 
@@ -6490,6 +6592,7 @@ Default property:
 {
   "modelType": "Llama 3.2 3B Chat",
   "chatFormat": "Llama 3.x",
+  "toolsInUserMessage": true,
   "toolCallParser": "llama3_json",
   "toolChoice": "auto",
   "parallelToolCalls": false,
@@ -6518,9 +6621,18 @@ Default property:
     /* Custom model split parts */
     customModelSplitParts?: number | DataLink
     /* Chat format */
-    chatFormat?: 'Llama 2' | 'Llama 3' | 'Llama 3.x' | 'Mistral v0.3' | 'Qwen 2' | DataLink
+    chatFormat?:
+      | 'Llama 2'
+      | 'Llama 3'
+      | 'Llama 3.x'
+      | 'Mistral v0.3'
+      | 'Qwen 2'
+      | 'Custom'
+      | DataLink
     /* Custom chat format template */
     customChatFormat?: string | DataLink
+    /* Put tools in user message */
+    toolsInUserMessage?: boolean | DataLink
     /* Prompt to generate */
     prompt?: string | DataLink
     /* Chat messages */
@@ -6536,7 +6648,7 @@ Default property:
     /* Stop words */
     stopWords?: Array<string | DataLink> | DataLink
     /* Tool call parser */
-    toolCallParser?: 'llama3_json' | 'mistral' | DataLink
+    toolCallParser?: 'llama3_json' | 'mistral' | 'hermes' | 'internlm' | 'phi4' | DataLink
     /* Tools for chat mode using OpenAI-compatible function calling format  
  Format: Array of objects with {type, function: {name, description, parameters}} structure  
  See: https://platform.openai.com/docs/guides/function-calling */
@@ -6915,6 +7027,11 @@ export type GeneratorAssistantActionAddMessage = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'payload'
         value?: {} | DataLink | EventProperty
@@ -7021,6 +7138,11 @@ export type GeneratorAssistantActionUpdateMessageAtIndex = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'payload'
         value?: {} | DataLink | EventProperty
@@ -7048,6 +7170,11 @@ export type GeneratorAssistantActionAddAudioMessage = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'useFileSearch'
         value?: boolean | DataLink | EventProperty
@@ -7147,6 +7274,11 @@ export type GeneratorAssistantActionUpdateAudioMessageAtIndex = ActionWithParams
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'payload'
         value?: {} | DataLink | EventProperty
@@ -7303,7 +7435,9 @@ Default property:
     cacheMessages?: boolean | DataLink
     /* LLM Generator (Supports `LLM (GGML)` and `OpenAI LLM` generators) */
     llmGeneratorId?: string | DataLink
-    /* LLM Live Policy. If the policy is `only-in-use`, the LLM context will be released when the assistant is not in use. */
+    /* LLM Live Policy. If the policy is `only-in-use`, the LLM context will be released when the assistant is not in use.  
+
+ Note: LLM (Qualcomm AI Engine) recommend use `manual` and loaded constantly. */
     llmLivePolicy?: 'only-in-use' | 'manual' | DataLink
     /* LLM main session key */
     llmSessionKey?: string | DataLink

package/utils/event-props.ts

@@ -724,6 +724,16 @@ export const templateEventPropsMap = {
     onContextStateChange: [
       'GENERATOR_ONNX_LLM_CONTEXT_STATE', // type: string
     ],
+    onFunctionCall: [
+      'GENERATOR_ONNX_LLM_FUNCTION_CALL_NAME', // type: string
+      'GENERATOR_ONNX_LLM_FUNCTION_CALL_ARGUMENTS', // type: object
+      'GENERATOR_ONNX_LLM_FUNCTION_CALL_DETAILS', // type: object
+    ],
+    onCompletionFinished: [
+      'GENERATOR_ONNX_LLM_COMPLETION_RESULT', // type: string
+      'GENERATOR_ONNX_LLM_COMPLETION_TOOL_CALLS', // type: array
+      'GENERATOR_ONNX_LLM_COMPLETION_FULL_CONTEXT', // type: string
+    ],
     onError: [
       'GENERATOR_ONNX_LLM_ERROR', // type: string
     ],
@@ -788,6 +798,7 @@ export const templateEventPropsMap = {
     onGenerate: [
       'GENERATOR_QNN_LLM_RESULT', // type: string
       'GENERATOR_QNN_LLM_FULL_CONTEXT', // type: string
+      'GENERATOR_QNN_LLM_TOOL_CALLS', // type: array
     ],
     onFunctionCall: [
       'GENERATOR_QNN_LLM_FUNCTION_CALL_NAME', // type: string