@fugood/bricks-project 2.22.0-beta.11 → 2.22.0-beta.12

package/compile/action-name-map.ts

@@ -552,6 +552,8 @@ export const templateActionNameMap = {
       prompt: 'GENERATOR_ONNX_LLM_PROMPT',
       chat: 'GENERATOR_ONNX_LLM_CHAT',
       images: 'GENERATOR_ONNX_LLM_IMAGES',
+      tools: 'GENERATOR_ONNX_LLM_TOOLS',
+      toolChoice: 'GENERATOR_ONNX_LLM_TOOL_CHOICE',
     },
   },
   GENERATOR_ONNX_STT: {
@@ -590,6 +592,14 @@ export const templateActionNameMap = {
     },
   },
   GENERATOR_LLM: {
+    GENERATOR_LLM_TOKENIZE: {
+      mode: 'GENERATOR_LLM_MODE',
+      messages: 'GENERATOR_LLM_MESSAGES',
+      prompt: 'GENERATOR_LLM_PROMPT',
+    },
+    GENERATOR_LLM_DETOKENIZE: {
+      tokens: 'GENERATOR_LLM_TOKENS',
+    },
     GENERATOR_LLM_PROCESS_PROMPT: {
       sessionKey: 'GENERATOR_LLM_SESSION_KEY',
       mode: 'GENERATOR_LLM_MODE',
@@ -598,6 +608,7 @@ export const templateActionNameMap = {
       parallelToolCalls: 'GENERATOR_LLM_PARALLEL_TOOL_CALLS',
       toolChoice: 'GENERATOR_LLM_TOOL_CHOICE',
       prompt: 'GENERATOR_LLM_PROMPT',
+      promptImagePaths: 'GENERATOR_LLM_PROMPT_IMAGE_PATHS',
       promptTemplateData: 'GENERATOR_LLM_PROMPT_TEMPLATE_DATA',
       promptTemplateType: 'GENERATOR_LLM_PROMPT_TEMPLATE_TYPE',
       responseFormat: 'GENERATOR_LLM_RESPONSE_FORMAT',
@@ -610,6 +621,7 @@ export const templateActionNameMap = {
       parallelToolCalls: 'GENERATOR_LLM_PARALLEL_TOOL_CALLS',
       toolChoice: 'GENERATOR_LLM_TOOL_CHOICE',
       prompt: 'GENERATOR_LLM_PROMPT',
+      promptImagePaths: 'GENERATOR_LLM_PROMPT_IMAGE_PATHS',
       promptTemplateData: 'GENERATOR_LLM_PROMPT_TEMPLATE_DATA',
       promptTemplateType: 'GENERATOR_LLM_PROMPT_TEMPLATE_TYPE',
       responseFormat: 'GENERATOR_LLM_RESPONSE_FORMAT',
@@ -650,6 +662,7 @@ export const templateActionNameMap = {
     GENERATOR_QNN_LLM_GENERATE: {
       prompt: 'GENERATOR_QNN_LLM_PROMPT',
       messages: 'GENERATOR_QNN_LLM_MESSAGES',
+      tools: 'GENERATOR_QNN_LLM_TOOLS',
     },
   },
   GENERATOR_OPENAI_LLM: {
@@ -676,6 +689,7 @@ export const templateActionNameMap = {
     GENERATOR_ASSISTANT_ADD_MESSAGE: {
       role: 'GENERATOR_ASSISTANT_ROLE',
       content: 'GENERATOR_ASSISTANT_CONTENT',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
       useFileSearch: 'GENERATOR_ASSISTANT_USE_FILE_SEARCH',
       filePath: 'GENERATOR_ASSISTANT_FILE_PATH',
@@ -699,12 +713,14 @@ export const templateActionNameMap = {
     GENERATOR_ASSISTANT_UPDATE_MESSAGE_AT_INDEX: {
       index: 'GENERATOR_ASSISTANT_INDEX',
       content: 'GENERATOR_ASSISTANT_CONTENT',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
     },
     GENERATOR_ASSISTANT_ADD_AUDIO_MESSAGE: {
       role: 'GENERATOR_ASSISTANT_ROLE',
       contentFile: 'GENERATOR_ASSISTANT_CONTENT_FILE',
       contentBase64: 'GENERATOR_ASSISTANT_CONTENT_BASE64',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       useFileSearch: 'GENERATOR_ASSISTANT_USE_FILE_SEARCH',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
       filePath: 'GENERATOR_ASSISTANT_FILE_PATH',
@@ -727,6 +743,7 @@ export const templateActionNameMap = {
       index: 'GENERATOR_ASSISTANT_INDEX',
       contentFile: 'GENERATOR_ASSISTANT_CONTENT_FILE',
       contentBase64: 'GENERATOR_ASSISTANT_CONTENT_BASE64',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
     },
     GENERATOR_ASSISTANT_REMOVE_MESSAGE_AT_INDEX: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fugood/bricks-project",
-  "version": "2.22.0-beta.11",
+  "version": "2.22.0-beta.12",
   "main": "index.ts",
   "scripts": {
     "build": "node scripts/build.js"
@@ -14,5 +14,5 @@
     "lodash": "^4.17.4",
     "uuid": "^8.3.1"
   },
-  "gitHead": "a6ee3595eab6b78d68c51effdeac7224e666134c"
+  "gitHead": "bf844dee8ddb0e8437c8795e0a44cafc48ef16d1"
 }

package/types/generators.ts

@@ -5000,6 +5000,16 @@ export type GeneratorOnnxLLMActionInfer = ActionWithParams & {
         value?: Array<any> | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'tools'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'toolChoice'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
   >
 }
 
@@ -5019,6 +5029,8 @@ Default property:
 {
   "model": "BricksDisplay/phi-1_5-q4",
   "modelType": "auto",
+  "toolCallParser": "llama3_json",
+  "toolChoice": "auto",
   "maxNewTokens": 256,
   "temperature": 0.7,
   "topK": 50,
@@ -5125,6 +5137,14 @@ Default property:
     messages?: Array<DataLink | {}> | DataLink
     /* Images with message to inference */
     images?: Array<string | DataLink> | DataLink
+    /* Tool call parser */
+    toolCallParser?: 'llama3_json' | 'mistral' | 'hermes' | 'internlm' | 'phi4' | DataLink
+    /* Tools for chat mode using OpenAI-compatible function calling format  
+ Format: Array of objects with {type, function: {name, description, parameters}} structure  
+ See: https://platform.openai.com/docs/guides/function-calling */
+    tools?: Array<{} | DataLink> | DataLink
+    /* Tool choice for chat mode */
+    toolChoice?: 'none' | 'auto' | DataLink
     /* Max new tokens to generate */
     maxNewTokens?: number | DataLink
     /* Temperature */
@@ -5159,6 +5179,10 @@ Default property:
   events?: {
     /* Event triggered when state change */
     onContextStateChange?: Array<EventAction>
+    /* Event triggered on get function call request */
+    onFunctionCall?: Array<EventAction>
+    /* Event triggered on completion finished */
+    onCompletionFinished?: Array<EventAction>
     /* Event triggered when error occurs */
     onError?: Array<EventAction>
   }
@@ -5169,6 +5193,8 @@ Default property:
     generated?: () => Data
     /* Full result of generation */
     fullResult?: () => Data
+    /* Last function call */
+    lastFunctionCall?: () => Data
   }
 }
 
@@ -5187,7 +5213,7 @@ export type GeneratorOnnxLLM = Generator &
               | SwitchCondData
               | {
                   __typename: 'SwitchCondInnerStateOutlet'
-                  outlet: 'contextState' | 'generated' | 'fullResult'
+                  outlet: 'contextState' | 'generated' | 'fullResult' | 'lastFunctionCall'
                   value: any
                 }
           }>
@@ -5878,6 +5904,43 @@ export type GeneratorLLMActionLoadModel = Action & {
   __actionName: 'GENERATOR_LLM_LOAD_MODEL'
 }
 
+/* Load multimodal (vision) model (PREVIEW FEATURE) */
+export type GeneratorLLMActionLoadMultimodalModel = Action & {
+  __actionName: 'GENERATOR_LLM_LOAD_MULTIMODAL_MODEL'
+}
+
+/* Tokenize the prompt */
+export type GeneratorLLMActionTokenize = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_TOKENIZE'
+  params?: Array<
+    | {
+        input: 'mode'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'messages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Detokenize the tokens to text */
+export type GeneratorLLMActionDetokenize = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_DETOKENIZE'
+  params?: Array<{
+    input: 'tokens'
+    value?: Array<any> | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
 /* Pre-process the prompt, this can speed up the completion action */
 export type GeneratorLLMActionProcessPrompt = ActionWithParams & {
   __actionName: 'GENERATOR_LLM_PROCESS_PROMPT'
@@ -5917,6 +5980,11 @@ export type GeneratorLLMActionProcessPrompt = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'promptImagePaths'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'promptTemplateData'
         value?: {} | DataLink | EventProperty
@@ -5974,6 +6042,11 @@ export type GeneratorLLMActionCompletion = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'promptImagePaths'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'promptTemplateData'
         value?: {} | DataLink | EventProperty
@@ -6154,6 +6227,11 @@ export type GeneratorLLMActionClearDownload = Action & {
   __actionName: 'GENERATOR_LLM_CLEAR_DOWNLOAD'
 }
 
+/* Release multimodal (vision) context (PREVIEW FEATURE) */
+export type GeneratorLLMActionReleaseMultimodalContext = Action & {
+  __actionName: 'GENERATOR_LLM_RELEASE_MULTIMODAL_CONTEXT'
+}
+
 /* Release context */
 export type GeneratorLLMActionReleaseContext = Action & {
   __actionName: 'GENERATOR_LLM_RELEASE_CONTEXT'
@@ -6231,6 +6309,14 @@ Default property:
     modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
     /* Hash of model */
     modelHash?: string | DataLink
+    /* Load multimodal (vision) context after model loaded (PREVIEW FEATURE) */
+    initMultimodal?: boolean | DataLink
+    /* The URL or path of mmproj file for multimodal vision support (PREVIEW FEATURE) */
+    mmprojUrl?: string | DataLink
+    /* Hash type of mmproj file (PREVIEW FEATURE) */
+    mmprojHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of mmproj file (PREVIEW FEATURE) */
+    mmprojHash?: string | DataLink
     /* Chat Template (Jinja format) to override the default template from model */
     chatTemplate?: string | DataLink
     /* Context size (0 ~ 4096) (Default to 512) */
@@ -6300,6 +6386,9 @@ Default property:
       | DataLink
     /* Prompt (text mode) */
     completionPrompt?: string | DataLink
+    /* Image paths to be used in the prompt template (PREVIEW FEATURE)  
+ In prompt, use `<__image__>` for position of image content */
+    completionPromptImagePaths?: Array<string | DataLink> | DataLink
     /* Data to be used in the prompt template (e.g. `Hello ${name}`). Supports nested data, such as `Hello ${user.name}`. */
     completionPromptTemplateData?: {} | DataLink
     /* The prompt template type */
@@ -6397,6 +6486,10 @@ Default property:
     sessions?: () => Data
     /* Is evaluating */
     isEvaluating?: () => Data
+    /* Tokenize result */
+    tokenizeResult?: () => Data
+    /* Detokenize result */
+    detokenizeResult?: () => Data
     /* Last formatted prompt (messages or prompt) */
     completionLastFormattedPrompt?: () => Data
     /* Last completion token */
@@ -6437,6 +6530,8 @@ export type GeneratorLLM = Generator &
                     | 'contextDetails'
                     | 'sessions'
                     | 'isEvaluating'
+                    | 'tokenizeResult'
+                    | 'detokenizeResult'
                     | 'completionLastFormattedPrompt'
                     | 'completionLastToken'
                     | 'completionResult'
@@ -6473,6 +6568,11 @@ export type GeneratorQnnLlmActionGenerate = ActionWithParams & {
         value?: Array<any> | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'tools'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
   >
 }
 
@@ -6492,6 +6592,7 @@ Default property:
 {
   "modelType": "Llama 3.2 3B Chat",
   "chatFormat": "Llama 3.x",
+  "toolsInUserMessage": true,
   "toolCallParser": "llama3_json",
   "toolChoice": "auto",
   "parallelToolCalls": false,
@@ -6520,9 +6621,18 @@ Default property:
     /* Custom model split parts */
     customModelSplitParts?: number | DataLink
     /* Chat format */
-    chatFormat?: 'Llama 2' | 'Llama 3' | 'Llama 3.x' | 'Mistral v0.3' | 'Qwen 2' | DataLink
+    chatFormat?:
+      | 'Llama 2'
+      | 'Llama 3'
+      | 'Llama 3.x'
+      | 'Mistral v0.3'
+      | 'Qwen 2'
+      | 'Custom'
+      | DataLink
     /* Custom chat format template */
     customChatFormat?: string | DataLink
+    /* Put tools in user message */
+    toolsInUserMessage?: boolean | DataLink
     /* Prompt to generate */
     prompt?: string | DataLink
     /* Chat messages */
@@ -6538,7 +6648,7 @@ Default property:
     /* Stop words */
     stopWords?: Array<string | DataLink> | DataLink
     /* Tool call parser */
-    toolCallParser?: 'llama3_json' | 'mistral' | DataLink
+    toolCallParser?: 'llama3_json' | 'mistral' | 'hermes' | 'internlm' | 'phi4' | DataLink
     /* Tools for chat mode using OpenAI-compatible function calling format  
  Format: Array of objects with {type, function: {name, description, parameters}} structure  
  See: https://platform.openai.com/docs/guides/function-calling */
@@ -6917,6 +7027,11 @@ export type GeneratorAssistantActionAddMessage = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'payload'
         value?: {} | DataLink | EventProperty
@@ -7023,6 +7138,11 @@ export type GeneratorAssistantActionUpdateMessageAtIndex = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'payload'
         value?: {} | DataLink | EventProperty
@@ -7050,6 +7170,11 @@ export type GeneratorAssistantActionAddAudioMessage = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'useFileSearch'
         value?: boolean | DataLink | EventProperty
@@ -7149,6 +7274,11 @@ export type GeneratorAssistantActionUpdateAudioMessageAtIndex = ActionWithParams
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'payload'
         value?: {} | DataLink | EventProperty
@@ -7305,7 +7435,9 @@ Default property:
     cacheMessages?: boolean | DataLink
     /* LLM Generator (Supports `LLM (GGML)` and `OpenAI LLM` generators) */
     llmGeneratorId?: string | DataLink
-    /* LLM Live Policy. If the policy is `only-in-use`, the LLM context will be released when the assistant is not in use. */
+    /* LLM Live Policy. If the policy is `only-in-use`, the LLM context will be released when the assistant is not in use.  
+
+ Note: LLM (Qualcomm AI Engine) recommend use `manual` and loaded constantly. */
     llmLivePolicy?: 'only-in-use' | 'manual' | DataLink
     /* LLM main session key */
     llmSessionKey?: string | DataLink

package/utils/event-props.ts

@@ -724,6 +724,16 @@ export const templateEventPropsMap = {
     onContextStateChange: [
       'GENERATOR_ONNX_LLM_CONTEXT_STATE', // type: string
     ],
+    onFunctionCall: [
+      'GENERATOR_ONNX_LLM_FUNCTION_CALL_NAME', // type: string
+      'GENERATOR_ONNX_LLM_FUNCTION_CALL_ARGUMENTS', // type: object
+      'GENERATOR_ONNX_LLM_FUNCTION_CALL_DETAILS', // type: object
+    ],
+    onCompletionFinished: [
+      'GENERATOR_ONNX_LLM_COMPLETION_RESULT', // type: string
+      'GENERATOR_ONNX_LLM_COMPLETION_TOOL_CALLS', // type: array
+      'GENERATOR_ONNX_LLM_COMPLETION_FULL_CONTEXT', // type: string
+    ],
     onError: [
       'GENERATOR_ONNX_LLM_ERROR', // type: string
     ],
@@ -788,6 +798,7 @@ export const templateEventPropsMap = {
     onGenerate: [
       'GENERATOR_QNN_LLM_RESULT', // type: string
       'GENERATOR_QNN_LLM_FULL_CONTEXT', // type: string
+      'GENERATOR_QNN_LLM_TOOL_CALLS', // type: array
     ],
     onFunctionCall: [
       'GENERATOR_QNN_LLM_FUNCTION_CALL_NAME', // type: string