@fugood/bricks-project 2.21.7 → 2.21.9

package/compile/action-name-map.ts

@@ -142,6 +142,13 @@ export const templateActionNameMap = {
     },
   },
 
+  BRICK_VIDEO: {
+    BRICK_VIDEO_SEEK: {
+      seekTime: 'BRICK_VIDEO_SEEK_TIME',
+      play: 'BRICK_VIDEO_PLAY',
+    },
+  },
+
   BRICK_SLIDESHOW: {
     BRICK_SLIDESHOW_JUMP_TO_INDEX: {
       index: 'BRICK_SLIDESHOW_INDEX',
@@ -231,7 +238,12 @@ export const templateActionNameMap = {
       index: 'BRICK_ITEMS_INDEX',
     },
   },
-
+  BRICK_LOTTIE: {
+    BRICK_LOTTIE_PLAY: {
+      startFrame: 'BRICK_LOTTIE_START_FRAME',
+      endFrame: 'BRICK_LOTTIE_END_FRAME',
+    },
+  },
   BRICK_RIVE: {
     BRICK_RIVE_PLAY: {
       animationName: 'BRICK_RIVE_ANIMATION_NAME',
@@ -257,6 +269,10 @@ export const templateActionNameMap = {
     BRICK_WEBVIEW_INJECT_JAVASCRIPT: {
       javascriptCode: 'BRICK_WEBVIEW_JAVASCRIPT_CODE',
     },
+    BRICK_WEBVIEW_QUERY_SELECTOR: {
+      querySelector: 'BRICK_WEBVIEW_QUERY_SELECTOR',
+      expression: 'BRICK_WEBVIEW_EXPRESSION',
+    },
   },
   BRICK_CAMERA: {
     BRICK_CAMERA_TAKE_PICTURE: {
@@ -592,6 +608,15 @@ export const templateActionNameMap = {
     },
   },
   GENERATOR_LLM: {
+    GENERATOR_LLM_TOKENIZE: {
+      mode: 'GENERATOR_LLM_MODE',
+      prompt: 'GENERATOR_LLM_PROMPT',
+      promptMediaPaths: 'GENERATOR_LLM_PROMPT_MEDIA_PATHS',
+      messages: 'GENERATOR_LLM_MESSAGES',
+    },
+    GENERATOR_LLM_DETOKENIZE: {
+      tokens: 'GENERATOR_LLM_TOKENS',
+    },
     GENERATOR_LLM_PROCESS_PROMPT: {
       sessionKey: 'GENERATOR_LLM_SESSION_KEY',
       mode: 'GENERATOR_LLM_MODE',
@@ -600,6 +625,7 @@ export const templateActionNameMap = {
       parallelToolCalls: 'GENERATOR_LLM_PARALLEL_TOOL_CALLS',
       toolChoice: 'GENERATOR_LLM_TOOL_CHOICE',
       prompt: 'GENERATOR_LLM_PROMPT',
+      promptMediaPaths: 'GENERATOR_LLM_PROMPT_MEDIA_PATHS',
       promptTemplateData: 'GENERATOR_LLM_PROMPT_TEMPLATE_DATA',
       promptTemplateType: 'GENERATOR_LLM_PROMPT_TEMPLATE_TYPE',
       responseFormat: 'GENERATOR_LLM_RESPONSE_FORMAT',
@@ -612,6 +638,7 @@ export const templateActionNameMap = {
       parallelToolCalls: 'GENERATOR_LLM_PARALLEL_TOOL_CALLS',
       toolChoice: 'GENERATOR_LLM_TOOL_CHOICE',
       prompt: 'GENERATOR_LLM_PROMPT',
+      promptMediaPaths: 'GENERATOR_LLM_PROMPT_MEDIA_PATHS',
       promptTemplateData: 'GENERATOR_LLM_PROMPT_TEMPLATE_DATA',
       promptTemplateType: 'GENERATOR_LLM_PROMPT_TEMPLATE_TYPE',
       responseFormat: 'GENERATOR_LLM_RESPONSE_FORMAT',
@@ -648,6 +675,11 @@ export const templateActionNameMap = {
       sessionCustomKey: 'GENERATOR_LLM_SESSION_CUSTOM_KEY',
     },
   },
+  GENERATOR_GGML_TTS: {
+    GENERATOR_GGML_TTS_GENERATE: {
+      text: 'GENERATOR_GGML_TTS_TEXT',
+    },
+  },
   GENERATOR_QNN_LLM: {
     GENERATOR_QNN_LLM_GENERATE: {
       prompt: 'GENERATOR_QNN_LLM_PROMPT',
@@ -679,6 +711,7 @@ export const templateActionNameMap = {
     GENERATOR_ASSISTANT_ADD_MESSAGE: {
       role: 'GENERATOR_ASSISTANT_ROLE',
       content: 'GENERATOR_ASSISTANT_CONTENT',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
       useFileSearch: 'GENERATOR_ASSISTANT_USE_FILE_SEARCH',
       filePath: 'GENERATOR_ASSISTANT_FILE_PATH',
@@ -702,12 +735,14 @@ export const templateActionNameMap = {
     GENERATOR_ASSISTANT_UPDATE_MESSAGE_AT_INDEX: {
       index: 'GENERATOR_ASSISTANT_INDEX',
       content: 'GENERATOR_ASSISTANT_CONTENT',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
     },
     GENERATOR_ASSISTANT_ADD_AUDIO_MESSAGE: {
       role: 'GENERATOR_ASSISTANT_ROLE',
       contentFile: 'GENERATOR_ASSISTANT_CONTENT_FILE',
       contentBase64: 'GENERATOR_ASSISTANT_CONTENT_BASE64',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       useFileSearch: 'GENERATOR_ASSISTANT_USE_FILE_SEARCH',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
       filePath: 'GENERATOR_ASSISTANT_FILE_PATH',
@@ -730,6 +765,7 @@ export const templateActionNameMap = {
       index: 'GENERATOR_ASSISTANT_INDEX',
       contentFile: 'GENERATOR_ASSISTANT_CONTENT_FILE',
       contentBase64: 'GENERATOR_ASSISTANT_CONTENT_BASE64',
+      image: 'GENERATOR_ASSISTANT_IMAGE',
       payload: 'GENERATOR_ASSISTANT_PAYLOAD',
     },
     GENERATOR_ASSISTANT_REMOVE_MESSAGE_AT_INDEX: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fugood/bricks-project",
-  "version": "2.21.7",
+  "version": "2.21.9",
   "main": "index.ts",
   "scripts": {
     "build": "node scripts/build.js"

package/types/bricks.ts

@@ -865,6 +865,43 @@ export type BrickIcon = Brick &
     >
   }
 
+/* Play the video */
+export type BrickVideoActionPlay = Action & {
+  __actionName: 'BRICK_VIDEO_PLAY'
+}
+
+/* Seek the video */
+export type BrickVideoActionSeek = ActionWithParams & {
+  __actionName: 'BRICK_VIDEO_SEEK'
+  params?: Array<
+    | {
+        input: 'seekTime'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'play'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Pause the video */
+export type BrickVideoActionPause = Action & {
+  __actionName: 'BRICK_VIDEO_PAUSE'
+}
+
+/* Replay the video */
+export type BrickVideoActionReplay = Action & {
+  __actionName: 'BRICK_VIDEO_REPLAY'
+}
+
+/* Stop the video */
+export type BrickVideoActionStop = Action & {
+  __actionName: 'BRICK_VIDEO_STOP'
+}
+
 interface BrickVideoDef {
   /*
 Default property:
@@ -2108,6 +2145,43 @@ export type BrickItems = Brick &
     >
   }
 
+/* Play animation */
+export type BrickLottieActionPlay = ActionWithParams & {
+  __actionName: 'BRICK_LOTTIE_PLAY'
+  params?: Array<
+    | {
+        input: 'startFrame'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'endFrame'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Pause animation */
+export type BrickLottieActionPause = Action & {
+  __actionName: 'BRICK_LOTTIE_PAUSE'
+}
+
+/* Resume animation */
+export type BrickLottieActionResume = Action & {
+  __actionName: 'BRICK_LOTTIE_RESUME'
+}
+
+/* Stop animation */
+export type BrickLottieActionStop = Action & {
+  __actionName: 'BRICK_LOTTIE_STOP'
+}
+
+/* Reset animation */
+export type BrickLottieActionReset = Action & {
+  __actionName: 'BRICK_LOTTIE_RESET'
+}
+
 interface BrickLottieDef {
   /*
 Default property:
@@ -2420,6 +2494,23 @@ export type BrickWebViewActionInjectJavascript = ActionWithParams & {
   }>
 }
 
+/* Query selector on the WebView */
+export type BrickWebViewActionQuerySelector = ActionWithParams & {
+  __actionName: 'BRICK_WEBVIEW_QUERY_SELECTOR'
+  params?: Array<
+    | {
+        input: 'querySelector'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'expression'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
 /* Do go forward on the WebView */
 export type BrickWebViewActionGoForward = Action & {
   __actionName: 'BRICK_WEBVIEW_GO_FORWARD'
@@ -2502,6 +2593,12 @@ Default property:
     /* Event of the webview on message by `window.ReactNativeWebView.postMessage` on you're injected javascript code */
     onMessage?: Array<EventAction>
   }
+  outlets?: {
+    /* The result of the query selector action */
+    queryResult?: () => Data
+    /* The error of the query selector action */
+    queryError?: () => Data
+  }
   animation?: AnimationBasicEvents & {
     onLoad?: Animation
     onError?: Animation
@@ -2523,7 +2620,7 @@ export type BrickWebView = Brick &
               | SwitchCondData
               | {
                   __typename: 'SwitchCondInnerStateOutlet'
-                  outlet: ''
+                  outlet: 'queryResult' | 'queryError'
                   value: any
                 }
           }>
@@ -2967,6 +3064,10 @@ Default property:
     type?: 'image' | 'video' | DataLink
     /* Default image to display when no generated image is available */
     defaultImage?: string | DataLink
+    /* The hash of the default image */
+    defaultImageHash?: string | DataLink
+    /* The type of the default image hash */
+    defaultImageHashType?: 'md5' | 'sha1' | 'sha256' | DataLink
     /* The Lottie animation to show while generating */
     loadingAnimation?: string | DataLink
     /* The Lottie animation to show when an error occurs */

package/types/data.ts

@@ -56,6 +56,11 @@ export type Data<T = any> = DataDef & {
           | 'rich-text-content'
           | 'sandbox-script'
           | 'llm-prompt'
+          | 'llm-messages'
+          | 'llm-tools'
+          | 'mcp-server-resources'
+          | 'mcp-server-tools'
+          | 'mcp-server-prompts'
       }
   value: T
 }
@@ -67,6 +72,7 @@ export type DataAssetKind = {
     | 'media-resource-audio'
     | 'media-resource-file'
     | 'lottie-file-uri'
+    | 'rive-file-uri'
     | 'ggml-model-asset'
     | 'gguf-model-asset'
     | 'binary-asset'

package/types/generators.ts

@@ -4848,6 +4848,10 @@ Default property:
   "model": "BricksDisplay/vits-eng",
   "modelType": "auto",
   "vocoderModel": "speecht5_hifigan",
+  "maxLength": 4096,
+  "temperature": 0.1,
+  "repetitionPenalty": 1.1,
+  "doSample": true,
   "outputType": "play",
   "cacheGenerated": true,
   "autoInferEnable": false,
@@ -4866,6 +4870,7 @@ Default property:
       | 'BricksDisplay/vits-eng'
       | 'BricksDisplay/vits-cmn'
       | 'BricksDisplay/ellie-Bert-VITS2'
+      | 'onnx-community/OuteTTS-1.0-0.6B-ONNX'
       | 'mms-tts-ara (NC)'
       | 'mms-tts-deu (NC)'
       | 'mms-tts-eng (NC)'
@@ -4881,7 +4886,7 @@ Default property:
       | 'speecht5_tts'
       | DataLink
     /* Model type */
-    modelType?: 'auto' | 'vits' | 'bert_vits2' | 'speecht5' | DataLink
+    modelType?: 'auto' | 'vits' | 'bert_vits2' | 'speecht5' | 'outetts-1.0' | DataLink
     /* Load quantized model (deprecated, use `quantizeType` instead) */
     quantized?: boolean | DataLink
     /* Quantize type */
@@ -4908,6 +4913,16 @@ Default property:
     speakerEmbedUrl?: string | DataLink
     /* MD5 checksum of `speakerEmbedUrl` */
     speakerEmbedMd5?: string | DataLink
+    /* Speaker config, for OuteTTS model */
+    speakerConfig?: {} | DataLink
+    /* Audio token generation max length */
+    maxLength?: number | DataLink
+    /* Audio token generation temperature */
+    temperature?: number | DataLink
+    /* Audio token generation repetition penalty */
+    repetitionPenalty?: number | DataLink
+    /* Use greedy sampling for audio token generation */
+    doSample?: boolean | DataLink
     /* Output mode */
     outputType?: 'play' | 'file' | DataLink
     /* Enable cache for generated audio */
@@ -5092,24 +5107,19 @@ Default property:
     /* Model type */
     modelType?:
       | 'auto'
-      | 'gpt2'
-      | 'gptj'
-      | 'gpt_bigcode'
-      | 'gpt_neo'
-      | 'gpt_neox'
-      | 'bloom'
-      | 'mpt'
-      | 'opt'
-      | 'llama'
-      | 'falcon'
-      | 'mistral'
+      | 'text-generation'
+      | 'qwen2-vl'
+      | 'paligemma'
+      | 'llava'
+      | 'llava_onevision'
+      | 'moondream1'
+      | 'florence2'
+      | 'idefics3'
+      | 'smolvlm'
+      | 'phi3_v'
       | 't5'
       | 'mt5'
       | 'longt5'
-      | 'phi'
-      | 'qwen2'
-      | 'stablelm'
-      | 'gemma'
       | DataLink
     /* Load quantized model (deprecated, use `quantizeType` instead) */
     quantized?: boolean | DataLink
@@ -5902,6 +5912,48 @@ export type GeneratorLLMActionLoadModel = Action & {
   __actionName: 'GENERATOR_LLM_LOAD_MODEL'
 }
 
+/* Load multimodal (vision) model (PREVIEW FEATURE) */
+export type GeneratorLLMActionLoadMultimodalModel = Action & {
+  __actionName: 'GENERATOR_LLM_LOAD_MULTIMODAL_MODEL'
+}
+
+/* Tokenize the prompt */
+export type GeneratorLLMActionTokenize = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_TOKENIZE'
+  params?: Array<
+    | {
+        input: 'mode'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'promptMediaPaths'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'messages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Detokenize the tokens to text */
+export type GeneratorLLMActionDetokenize = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_DETOKENIZE'
+  params?: Array<{
+    input: 'tokens'
+    value?: Array<any> | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
 /* Pre-process the prompt, this can speed up the completion action */
 export type GeneratorLLMActionProcessPrompt = ActionWithParams & {
   __actionName: 'GENERATOR_LLM_PROCESS_PROMPT'
@@ -5941,6 +5993,11 @@ export type GeneratorLLMActionProcessPrompt = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'promptMediaPaths'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'promptTemplateData'
         value?: {} | DataLink | EventProperty
@@ -5998,6 +6055,11 @@ export type GeneratorLLMActionCompletion = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'promptMediaPaths'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'promptTemplateData'
         value?: {} | DataLink | EventProperty
@@ -6178,6 +6240,11 @@ export type GeneratorLLMActionClearDownload = Action & {
   __actionName: 'GENERATOR_LLM_CLEAR_DOWNLOAD'
 }
 
+/* Release multimodal (vision) context (PREVIEW FEATURE) */
+export type GeneratorLLMActionReleaseMultimodalContext = Action & {
+  __actionName: 'GENERATOR_LLM_RELEASE_MULTIMODAL_CONTEXT'
+}
+
 /* Release context */
 export type GeneratorLLMActionReleaseContext = Action & {
   __actionName: 'GENERATOR_LLM_RELEASE_CONTEXT'
@@ -6255,6 +6322,14 @@ Default property:
     modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
     /* Hash of model */
     modelHash?: string | DataLink
+    /* Load multimodal (vision) context after model loaded (PREVIEW FEATURE) */
+    initMultimodal?: boolean | DataLink
+    /* The URL or path of mmproj file for multimodal vision support (PREVIEW FEATURE) */
+    mmprojUrl?: string | DataLink
+    /* Hash type of mmproj file (PREVIEW FEATURE) */
+    mmprojHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of mmproj file (PREVIEW FEATURE) */
+    mmprojHash?: string | DataLink
     /* Chat Template (Jinja format) to override the default template from model */
     chatTemplate?: string | DataLink
     /* Context size (0 ~ 4096) (Default to 512) */
@@ -6324,6 +6399,9 @@ Default property:
       | DataLink
     /* Prompt (text mode) */
     completionPrompt?: string | DataLink
+    /* Media paths to be used in the prompt template (PREVIEW FEATURE)  
+ In prompt, use `<__media__>` for position of media content */
+    completionPromptMediaPaths?: Array<string | DataLink> | DataLink
     /* Data to be used in the prompt template (e.g. `Hello ${name}`). Supports nested data, such as `Hello ${user.name}`. */
     completionPromptTemplateData?: {} | DataLink
     /* The prompt template type */
@@ -6421,6 +6499,10 @@ Default property:
     sessions?: () => Data
     /* Is evaluating */
     isEvaluating?: () => Data
+    /* Tokenize result */
+    tokenizeResult?: () => Data
+    /* Detokenize result */
+    detokenizeResult?: () => Data
     /* Last formatted prompt (messages or prompt) */
     completionLastFormattedPrompt?: () => Data
     /* Last completion token */
@@ -6461,6 +6543,8 @@ export type GeneratorLLM = Generator &
                     | 'contextDetails'
                     | 'sessions'
                     | 'isEvaluating'
+                    | 'tokenizeResult'
+                    | 'detokenizeResult'
                     | 'completionLastFormattedPrompt'
                     | 'completionLastToken'
                     | 'completionResult'
@@ -6473,6 +6557,165 @@ export type GeneratorLLM = Generator &
     >
   }
 
+/* Load the model */
+export type GeneratorGGMLTTSActionLoadModel = Action & {
+  __actionName: 'GENERATOR_GGML_TTS_LOAD_MODEL'
+}
+
+/* Generate audio */
+export type GeneratorGGMLTTSActionGenerate = ActionWithParams & {
+  __actionName: 'GENERATOR_GGML_TTS_GENERATE'
+  params?: Array<{
+    input: 'text'
+    value?: string | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Clean cache */
+export type GeneratorGGMLTTSActionCleanCache = Action & {
+  __actionName: 'GENERATOR_GGML_TTS_CLEAN_CACHE'
+}
+
+/* Release context */
+export type GeneratorGGMLTTSActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_GGML_TTS_RELEASE_CONTEXT'
+}
+
+interface GeneratorGGMLTTSDef {
+  /*
+Default property:
+{
+  "vocoderUrl": "https://huggingface.co/ggml-org/WavTokenizer/resolve/main/WavTokenizer-Large-75-F16.gguf",
+  "vocoderHashType": "sha256",
+  "vocoderHash": "2356baa8631cc2995ea3465196a017a2733600d849a91180c0f97fa7fb375bbe",
+  "outputType": "play",
+  "cacheGenerated": true,
+  "autoInferEnable": false,
+  "softBreakRegex": "^[^\\r\\n\\t\\f\\v]*([\\r\\n]+|[。！？!?.]\\B)",
+  "hardBreakTime": 500,
+  "completionTemperature": 0.1,
+  "completionRepetitionPenalty": 1.1,
+  "completionTopK": 40,
+  "completionTopP": 0.9,
+  "completionMinP": 0.05,
+  "useGuideToken": false,
+  "contextSize": 4096,
+  "batchSize": 512,
+  "maxThreads": 1,
+  "accelVariant": "default",
+  "mainGpu": 0,
+  "gpuLayers": 0,
+  "useMlock": true,
+  "useMmap": true,
+  "useFlashAttn": false
+}
+  */
+  property?: {
+    /* Initialize the TTS context on generator initialization */
+    init?: boolean | DataLink
+    /* The URL or path of model  
+ We used GGUF format model, please refer to https://github.com/ggerganov/llama.cpp/tree/master#description */
+    modelUrl?: string | DataLink
+    /* Hash type of model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of model */
+    modelHash?: string | DataLink
+    /* The URL or path of vocoder model */
+    vocoderUrl?: string | DataLink
+    /* Hash type of vocoder model */
+    vocoderHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of vocoder model */
+    vocoderHash?: string | DataLink
+    /* Output mode */
+    outputType?: 'play' | 'file' | DataLink
+    /* Enable cache for generated audio */
+    cacheGenerated?: boolean | DataLink
+    /* Text to generate */
+    prompt?: string | DataLink
+    /* Speaker JSON */
+    speaker?: {} | DataLink
+    /* Auto inference when prompt changes */
+    autoInferEnable?: boolean | DataLink
+    /* Segmentation rule for auto inference */
+    softBreakRegex?: string | DataLink
+    /* Time to force inference when softBreakRegex is not satisfied */
+    hardBreakTime?: number | DataLink
+    /* Temperature */
+    completionTemperature?: number | DataLink
+    /* Repetition Penalty */
+    completionRepetitionPenalty?: number | DataLink
+    /* Top K sampling */
+    completionTopK?: number | DataLink
+    /* Top P sampling */
+    completionTopP?: number | DataLink
+    /* Min P sampling */
+    completionMinP?: number | DataLink
+    /* Set the random number generator (RNG) seed (default: -1, -1 = random seed) */
+    completionSeed?: number | DataLink
+    /* Enable guide token to help prevent hallucinations by forcing the TTS to use the correct words. */
+    useGuideToken?: boolean | DataLink
+    /* Context size, for OutTTS recommended 4096 ~ 8192 (Default to 4096) */
+    contextSize?: number | DataLink
+    /* Logical batch size for prompt processing */
+    batchSize?: number | DataLink
+    /* Number of threads */
+    maxThreads?: number | DataLink
+    /* Accelerator variant (Only for desktop)  
+ `default` - CPU / Metal (macOS)  
+ `vulkan` - Use Vulkan  
+ `cuda` - Use CUDA */
+    accelVariant?: 'default' | 'vulkan' | 'cuda' | DataLink
+    /* Main GPU index */
+    mainGpu?: number | DataLink
+    /* Number of GPU layers (NOTE: Currently not supported for Android) */
+    gpuLayers?: number | DataLink
+    /* Use memory lock */
+    useMlock?: boolean | DataLink
+    /* Use mmap */
+    useMmap?: boolean | DataLink
+    /* Use Flash Attention for inference (Recommended with GPU enabled) */
+    useFlashAttn?: boolean | DataLink
+  }
+  events?: {
+    /* Event triggered when state change */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Generated audio file */
+    generatedAudio?: () => Data
+    /* Generated audio file is playing */
+    generatedAudioPlaying?: () => Data
+  }
+}
+
+/* Local Text-to-Speech (TTS) inference based on GGML and [llama.cpp](https://github.com/ggerganov/llama.cpp)  
+ You can use any converted model on HuggingFace. */
+export type GeneratorGGMLTTS = Generator &
+  GeneratorGGMLTTSDef & {
+    templateKey: 'GENERATOR_GGML_TTS'
+    switches: Array<
+      SwitchDef &
+        GeneratorGGMLTTSDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'contextState' | 'generatedAudio' | 'generatedAudioPlaying'
+                  value: any
+                }
+          }>
+        }
+    >
+  }
+
 /* Load the model */
 export type GeneratorQnnLlmActionLoadModel = Action & {
   __actionName: 'GENERATOR_QNN_LLM_LOAD_MODEL'
@@ -6521,6 +6764,7 @@ Default property:
 {
   "modelType": "Llama 3.2 3B Chat",
   "chatFormat": "Llama 3.x",
+  "toolsInUserMessage": true,
   "toolCallParser": "llama3_json",
   "toolChoice": "auto",
   "parallelToolCalls": false,
@@ -6549,9 +6793,18 @@ Default property:
     /* Custom model split parts */
     customModelSplitParts?: number | DataLink
     /* Chat format */
-    chatFormat?: 'Llama 2' | 'Llama 3' | 'Llama 3.x' | 'Mistral v0.3' | 'Qwen 2' | DataLink
+    chatFormat?:
+      | 'Llama 2'
+      | 'Llama 3'
+      | 'Llama 3.x'
+      | 'Mistral v0.3'
+      | 'Qwen 2'
+      | 'Custom'
+      | DataLink
     /* Custom chat format template */
     customChatFormat?: string | DataLink
+    /* Put tools in user message */
+    toolsInUserMessage?: boolean | DataLink
     /* Prompt to generate */
     prompt?: string | DataLink
     /* Chat messages */
@@ -6712,7 +6965,7 @@ interface GeneratorOpenAILLMDef {
 Default property:
 {
   "apiEndpoint": "https://api.openai.com/v1",
-  "model": "gpt-4o-mini",
+  "model": "gpt-4o",
   "completionMessages": [
     {
       "role": "system",
@@ -6722,8 +6975,6 @@ Default property:
   "completionMaxTokens": 1024,
   "completionTemperature": 1,
   "completionTopP": 1,
-  "completionFrequencyPenalty": 0,
-  "completionPresencePenalty": 0,
   "completionStop": []
 }
   */
@@ -6812,7 +7063,11 @@ Default property:
  - Compatible with OpenAI API format  
  - Supports function calling  
  - Streaming responses  
- - Custom API endpoints */
+ - Custom API endpoints, like  
+   - OpenAI API: https://platform.openai.com/docs/guides/text?api-mode=chat  
+   - Anthropic API: https://docs.anthropic.com/en/api/openai-sdk  
+   - Gemini API: https://ai.google.dev/gemini-api/docs/openai  
+   - llama.cpp server: https://github.com/ggml-org/llama.cpp/tree/master/tools/server */
 export type GeneratorOpenAILLM = Generator &
   GeneratorOpenAILLMDef & {
     templateKey: 'GENERATOR_OPENAI_LLM'
@@ -6946,6 +7201,11 @@ export type GeneratorAssistantActionAddMessage = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'payload'
         value?: {} | DataLink | EventProperty
@@ -7052,6 +7312,11 @@ export type GeneratorAssistantActionUpdateMessageAtIndex = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'payload'
         value?: {} | DataLink | EventProperty
@@ -7079,6 +7344,11 @@ export type GeneratorAssistantActionAddAudioMessage = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'useFileSearch'
         value?: boolean | DataLink | EventProperty
@@ -7178,6 +7448,11 @@ export type GeneratorAssistantActionUpdateAudioMessageAtIndex = ActionWithParams
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'payload'
         value?: {} | DataLink | EventProperty

package/utils/event-props.ts

@@ -806,6 +806,14 @@ export const templateEventPropsMap = {
       'GENERATOR_LLM_COMPLETION_FUNCTION_DETAILS', // type: object
     ],
   },
+  GENERATOR_GGML_TTS: {
+    onContextStateChange: [
+      'GENERATOR_GGML_TTS_CONTEXT_STATE', // type: string
+    ],
+    onError: [
+      'GENERATOR_GGML_TTS_ERROR', // type: string
+    ],
+  },
   GENERATOR_QNN_LLM: {
     onContextStateChange: [
       'GENERATOR_QNN_LLM_CONTEXT_STATE', // type: string