@fugood/bricks-project 2.22.0-beta.14 → 2.22.0-beta.16

package/compile/action-name-map.ts

@@ -100,6 +100,9 @@ export const templateActionNameMap = {
       payloadType: 'CHANNEL_PUBLISH_PAYLOAD_TYPE',
       payload: 'CHANNEL_PUBLISH_PAYLOAD',
     },
+    DELAY: {
+      time: 'DELAY_TIME',
+    },
     USE_SHARE_APPLICATION: {
       applicationId: 'APPLICATION_ID',
       releaseVersion: 'RELEASE_VERSION',
@@ -142,6 +145,13 @@ export const templateActionNameMap = {
     },
   },
 
+  BRICK_VIDEO: {
+    BRICK_VIDEO_SEEK: {
+      seekTime: 'BRICK_VIDEO_SEEK_TIME',
+      play: 'BRICK_VIDEO_PLAY',
+    },
+  },
+
   BRICK_SLIDESHOW: {
     BRICK_SLIDESHOW_JUMP_TO_INDEX: {
       index: 'BRICK_SLIDESHOW_INDEX',
@@ -231,7 +241,12 @@ export const templateActionNameMap = {
       index: 'BRICK_ITEMS_INDEX',
     },
   },
-
+  BRICK_LOTTIE: {
+    BRICK_LOTTIE_PLAY: {
+      startFrame: 'BRICK_LOTTIE_START_FRAME',
+      endFrame: 'BRICK_LOTTIE_END_FRAME',
+    },
+  },
   BRICK_RIVE: {
     BRICK_RIVE_PLAY: {
       animationName: 'BRICK_RIVE_ANIMATION_NAME',
@@ -595,6 +610,26 @@ export const templateActionNameMap = {
       realtimeVadFreqThold: 'GENERATOR_SPEECH_INFERENCE_REALTIME_VAD_FREQ_THOLD',
     },
   },
+  GENERATOR_VAD_INFERENCE: {
+    GENERATOR_VAD_INFERENCE_DETECT_FILE: {
+      fileUrl: 'GENERATOR_VAD_INFERENCE_FILE_URL',
+      threshold: 'GENERATOR_VAD_INFERENCE_THRESHOLD',
+      minSpeechDurationMs: 'GENERATOR_VAD_INFERENCE_MIN_SPEECH_DURATION_MS',
+      minSilenceDurationMs: 'GENERATOR_VAD_INFERENCE_MIN_SILENCE_DURATION_MS',
+      maxSpeechDurationS: 'GENERATOR_VAD_INFERENCE_MAX_SPEECH_DURATION_S',
+      speechPadMs: 'GENERATOR_VAD_INFERENCE_SPEECH_PAD_MS',
+      samplesOverlap: 'GENERATOR_VAD_INFERENCE_SAMPLES_OVERLAP',
+    },
+    GENERATOR_VAD_INFERENCE_DETECT_DATA: {
+      data: 'GENERATOR_VAD_INFERENCE_DATA',
+      threshold: 'GENERATOR_VAD_INFERENCE_THRESHOLD',
+      minSpeechDurationMs: 'GENERATOR_VAD_INFERENCE_MIN_SPEECH_DURATION_MS',
+      minSilenceDurationMs: 'GENERATOR_VAD_INFERENCE_MIN_SILENCE_DURATION_MS',
+      maxSpeechDurationS: 'GENERATOR_VAD_INFERENCE_MAX_SPEECH_DURATION_S',
+      speechPadMs: 'GENERATOR_VAD_INFERENCE_SPEECH_PAD_MS',
+      samplesOverlap: 'GENERATOR_VAD_INFERENCE_SAMPLES_OVERLAP',
+    },
+  },
   GENERATOR_LLM: {
     GENERATOR_LLM_TOKENIZE: {
       mode: 'GENERATOR_LLM_MODE',
@@ -612,6 +647,7 @@ export const templateActionNameMap = {
       tools: 'GENERATOR_LLM_TOOLS',
       parallelToolCalls: 'GENERATOR_LLM_PARALLEL_TOOL_CALLS',
       toolChoice: 'GENERATOR_LLM_TOOL_CHOICE',
+      enableThinking: 'GENERATOR_LLM_ENABLE_THINKING',
       prompt: 'GENERATOR_LLM_PROMPT',
       promptMediaPaths: 'GENERATOR_LLM_PROMPT_MEDIA_PATHS',
       promptTemplateData: 'GENERATOR_LLM_PROMPT_TEMPLATE_DATA',
@@ -625,6 +661,7 @@ export const templateActionNameMap = {
       tools: 'GENERATOR_LLM_TOOLS',
       parallelToolCalls: 'GENERATOR_LLM_PARALLEL_TOOL_CALLS',
       toolChoice: 'GENERATOR_LLM_TOOL_CHOICE',
+      enableThinking: 'GENERATOR_LLM_ENABLE_THINKING',
       prompt: 'GENERATOR_LLM_PROMPT',
       promptMediaPaths: 'GENERATOR_LLM_PROMPT_MEDIA_PATHS',
       promptTemplateData: 'GENERATOR_LLM_PROMPT_TEMPLATE_DATA',
@@ -663,6 +700,17 @@ export const templateActionNameMap = {
       sessionCustomKey: 'GENERATOR_LLM_SESSION_CUSTOM_KEY',
     },
   },
+  GENERATOR_GGML_TTS: {
+    GENERATOR_GGML_TTS_GENERATE: {
+      text: 'GENERATOR_GGML_TTS_TEXT',
+    },
+  },
+  GENERATOR_RERANKER: {
+    GENERATOR_RERANKER_RERANK: {
+      query: 'GENERATOR_RERANKER_QUERY',
+      documents: 'GENERATOR_RERANKER_DOCUMENTS',
+    },
+  },
   GENERATOR_QNN_LLM: {
     GENERATOR_QNN_LLM_GENERATE: {
       prompt: 'GENERATOR_QNN_LLM_PROMPT',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fugood/bricks-project",
-  "version": "2.22.0-beta.14",
+  "version": "2.22.0-beta.16",
   "main": "index.ts",
   "scripts": {
     "build": "node scripts/build.js"
@@ -14,5 +14,5 @@
     "lodash": "^4.17.4",
     "uuid": "^8.3.1"
   },
-  "gitHead": "4b62185a9ec82561db90d85b8281de9e67f978ae"
+  "gitHead": "d8231fa6787b8d95bc2bec94a86ceba596cf57f0"
 }

package/tools/postinstall.ts

@@ -1,15 +1,7 @@
 import { $ } from 'bun'
 import { stat, readFile, writeFile } from 'fs/promises'
-const cwd = process.cwd()
-
-const libFiles = ['types', 'utils', 'index.ts']
 
-await $`mkdir -p ${cwd}/project`
-for (const file of libFiles) {
-  await $`cp -r ${__dirname}/../${file} ${cwd}/project`
-}
-
-console.log('Copied files to project/')
+const cwd = process.cwd()
 
 async function exists(f: string) {
   try {
@@ -20,6 +12,21 @@ async function exists(f: string) {
   }
 }
 
+// handle flag --skip-copy
+const skipCopyProject = process.argv.includes('--skip-copy-project')
+if (skipCopyProject) {
+  console.log('Skipping copy of files to project/')
+} else {
+
+  const libFiles = ['types', 'utils', 'index.ts']
+  
+  await $`mkdir -p ${cwd}/project`
+  for (const file of libFiles) {
+    await $`cp -r ${__dirname}/../${file} ${cwd}/project`
+  }
+  console.log('Copied files to project/')
+}
+
 const projectMcpServer = {
   command: 'bun',
   args: [`${cwd}/node_modules/@fugood/bricks-project/tools/mcp-server.ts`],

package/types/bricks.ts

@@ -865,6 +865,43 @@ export type BrickIcon = Brick &
     >
   }
 
+/* Play the video */
+export type BrickVideoActionPlay = Action & {
+  __actionName: 'BRICK_VIDEO_PLAY'
+}
+
+/* Seek the video */
+export type BrickVideoActionSeek = ActionWithParams & {
+  __actionName: 'BRICK_VIDEO_SEEK'
+  params?: Array<
+    | {
+        input: 'seekTime'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'play'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Pause the video */
+export type BrickVideoActionPause = Action & {
+  __actionName: 'BRICK_VIDEO_PAUSE'
+}
+
+/* Replay the video */
+export type BrickVideoActionReplay = Action & {
+  __actionName: 'BRICK_VIDEO_REPLAY'
+}
+
+/* Stop the video */
+export type BrickVideoActionStop = Action & {
+  __actionName: 'BRICK_VIDEO_STOP'
+}
+
 interface BrickVideoDef {
   /*
 Default property:
@@ -2108,6 +2145,43 @@ export type BrickItems = Brick &
     >
   }
 
+/* Play animation */
+export type BrickLottieActionPlay = ActionWithParams & {
+  __actionName: 'BRICK_LOTTIE_PLAY'
+  params?: Array<
+    | {
+        input: 'startFrame'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'endFrame'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Pause animation */
+export type BrickLottieActionPause = Action & {
+  __actionName: 'BRICK_LOTTIE_PAUSE'
+}
+
+/* Resume animation */
+export type BrickLottieActionResume = Action & {
+  __actionName: 'BRICK_LOTTIE_RESUME'
+}
+
+/* Stop animation */
+export type BrickLottieActionStop = Action & {
+  __actionName: 'BRICK_LOTTIE_STOP'
+}
+
+/* Reset animation */
+export type BrickLottieActionReset = Action & {
+  __actionName: 'BRICK_LOTTIE_RESET'
+}
+
 interface BrickLottieDef {
   /*
 Default property:
@@ -2990,6 +3064,10 @@ Default property:
     type?: 'image' | 'video' | DataLink
     /* Default image to display when no generated image is available */
     defaultImage?: string | DataLink
+    /* The hash of the default image */
+    defaultImageHash?: string | DataLink
+    /* The type of the default image hash */
+    defaultImageHashType?: 'md5' | 'sha1' | 'sha256' | DataLink
     /* The Lottie animation to show while generating */
     loadingAnimation?: string | DataLink
     /* The Lottie animation to show when an error occurs */

package/types/common.ts

@@ -65,7 +65,7 @@ export type ItemBrickID = string
 
 export type EventAction = {
   handler: 'system' | (() => Brick | Generator) | SubspaceID | ItemBrickID
-  action: Action
+  action: ActionWithParams | ActionWithDataParams
   waitAsync?: boolean
 }
 

package/types/generators.ts

@@ -182,8 +182,8 @@ export type GeneratorFileActionReadContent = ActionWithParams & {
 }
 
 /* Delete */
-export type GeneratorFileActionGeneratorDeleteFile = Action & {
-  __actionName: 'GENERATOR_DELETE_FILE'
+export type GeneratorFileActionDelete = Action & {
+  __actionName: 'GENERATOR_FILE_DELETE'
 }
 
 /* Append (Currently only support text file) */
@@ -4850,8 +4850,13 @@ Default property:
   "model": "BricksDisplay/vits-eng",
   "modelType": "auto",
   "vocoderModel": "speecht5_hifigan",
+  "maxLength": 4096,
+  "temperature": 0.1,
+  "repetitionPenalty": 1.1,
+  "doSample": true,
   "outputType": "play",
   "cacheGenerated": true,
+  "speed": 1,
   "autoInferEnable": false,
   "softBreakRegex": "^[^\\r\\n\\t\\f\\v]*([\\r\\n]+|[。！？!?.]\\B)",
   "hardBreakTime": 500,
@@ -4868,6 +4873,9 @@ Default property:
       | 'BricksDisplay/vits-eng'
       | 'BricksDisplay/vits-cmn'
       | 'BricksDisplay/ellie-Bert-VITS2'
+      | 'onnx-community/Kokoro-82M-v1.0-ONNX'
+      | 'onnx-community/Kokoro-82M-v1.1-zh-ONNX'
+      | 'onnx-community/OuteTTS-1.0-0.6B-ONNX'
       | 'mms-tts-ara (NC)'
       | 'mms-tts-deu (NC)'
       | 'mms-tts-eng (NC)'
@@ -4883,7 +4891,14 @@ Default property:
       | 'speecht5_tts'
       | DataLink
     /* Model type */
-    modelType?: 'auto' | 'vits' | 'bert_vits2' | 'speecht5' | DataLink
+    modelType?:
+      | 'auto'
+      | 'vits'
+      | 'bert_vits2'
+      | 'style_text_to_speech_2'
+      | 'speecht5'
+      | 'outetts-1.0'
+      | DataLink
     /* Load quantized model (deprecated, use `quantizeType` instead) */
     quantized?: boolean | DataLink
     /* Quantize type */
@@ -4906,14 +4921,26 @@ Default property:
     /* Custom vocoder model  
  Choose model from https://huggingface.co/models?library=transformers.js&other=hifigan */
     customVocoderModel?: string | DataLink
-    /* XVector speaker embedding for HiFi-GAN */
+    /* Speaker embedding, for SpeechT5 or StyleTTS (Kokoro) */
     speakerEmbedUrl?: string | DataLink
     /* MD5 checksum of `speakerEmbedUrl` */
     speakerEmbedMd5?: string | DataLink
+    /* Speaker config, for OuteTTS model */
+    speakerConfig?: {} | DataLink
+    /* Audio token generation max length */
+    maxLength?: number | DataLink
+    /* Audio token generation temperature */
+    temperature?: number | DataLink
+    /* Audio token generation repetition penalty */
+    repetitionPenalty?: number | DataLink
+    /* Use greedy sampling for audio token generation */
+    doSample?: boolean | DataLink
     /* Output mode */
     outputType?: 'play' | 'file' | DataLink
     /* Enable cache for generated audio */
     cacheGenerated?: boolean | DataLink
+    /* Speed of the generated audio, for StyleTTS (Kokoro) */
+    speed?: number | DataLink
     /* Text to generate */
     prompt?: string | DataLink
     /* Auto inference when prompt changes */
@@ -5894,6 +5921,207 @@ export type GeneratorSpeechInference = Generator &
     >
   }
 
+/* Load the model */
+export type GeneratorVadInferenceActionLoadModel = Action & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_LOAD_MODEL'
+}
+
+/* Detect speech in audio file. You can provide `File URL` property, if not provided, it will use the default `File URL` */
+export type GeneratorVadInferenceActionDetectFile = ActionWithParams & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_DETECT_FILE'
+  params?: Array<
+    | {
+        input: 'fileUrl'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSpeechDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSilenceDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxSpeechDurationS'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'speechPadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'samplesOverlap'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Detect speech in audio data. Currently only support base64 encoded audio data (16-bit PCM, mono, 16kHz) */
+export type GeneratorVadInferenceActionDetectData = ActionWithParams & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_DETECT_DATA'
+  params?: Array<
+    | {
+        input: 'data'
+        value?: any | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSpeechDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSilenceDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxSpeechDurationS'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'speechPadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'samplesOverlap'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Clear downloaded files (model, audio) & current jobs */
+export type GeneratorVadInferenceActionClearDownload = Action & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_CLEAR_DOWNLOAD'
+}
+
+/* Release context */
+export type GeneratorVadInferenceActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_RELEASE_CONTEXT'
+}
+
+interface GeneratorVadInferenceDef {
+  /*
+Default property:
+{
+  "init": false,
+  "modelName": "silero-v5.1.2",
+  "modelUseGPU": true,
+  "modelThreads": 4,
+  "detectThreshold": 0.5,
+  "detectMinSpeechDurationMs": 250,
+  "detectMinSilenceDurationMs": 100,
+  "detectMaxSpeechDurationS": 30,
+  "detectSpeechPadMs": 30,
+  "detectSamplesOverlap": 0.1
+}
+  */
+  property?: {
+    /* Initialize the VAD context on generator initialization  
+ Please note that it will take some RAM depending on the model size */
+    init?: boolean | DataLink
+    /* Use model name, currently only supports the Silero VAD model.  
+ The model download progress will be done in preload stage or  the generator initialization stage.  
+ You can also choose `custom` option and set `Model URL` and `Model SHA1` to use your own model */
+    modelName?: 'custom' | 'silero-v5.1.2' | DataLink
+    /* The URL or path of model  
+ We used `ggml` format model, please refer to https://huggingface.co/ggml-org/whisper-vad */
+    modelUrl?: string | DataLink
+    /* Hash type of model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of model */
+    modelHash?: string | DataLink
+    /* Use GPU Acceleration for inference. Currently iOS only. */
+    modelUseGPU?: boolean | DataLink
+    /* Number of threads to use for processing */
+    modelThreads?: number | DataLink
+    /* Speech probability threshold (0.0-1.0) */
+    detectThreshold?: number | DataLink
+    /* Minimum speech duration in milliseconds */
+    detectMinSpeechDurationMs?: number | DataLink
+    /* Minimum silence duration in milliseconds */
+    detectMinSilenceDurationMs?: number | DataLink
+    /* Maximum speech duration in seconds */
+    detectMaxSpeechDurationS?: number | DataLink
+    /* Padding around speech segments in milliseconds */
+    detectSpeechPadMs?: number | DataLink
+    /* Overlap between analysis windows (0.0-1.0) */
+    detectSamplesOverlap?: number | DataLink
+    /* The file URL or path to be analyzed.  
+ It only supported `wav` format with 16kHz sample rate & single (mono) channel */
+    detectFileUrl?: string | DataLink
+    /* MD5 of file to be analyzed */
+    detectFileMd5?: string | DataLink
+  }
+  events?: {
+    /* Event triggered when load is done */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+    /* Event triggered when got detection result */
+    onDetected?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Context details */
+    contextDetails?: () => Data
+    /* Is detecting */
+    isDetecting?: () => Data
+    /* Detection segments result */
+    detectionSegments?: () => Data
+    /* Detection details */
+    detectionDetails?: () => Data
+  }
+}
+
+/* Local Voice Activity Detection (VAD) inference based on GGML and [whisper.rn](https://github.com/mybigday/whisper.rn) */
+export type GeneratorVadInference = Generator &
+  GeneratorVadInferenceDef & {
+    templateKey: 'GENERATOR_VAD_INFERENCE'
+    switches: Array<
+      SwitchDef &
+        GeneratorVadInferenceDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'contextDetails'
+                    | 'isDetecting'
+                    | 'detectionSegments'
+                    | 'detectionDetails'
+                  value: any
+                }
+          }>
+        }
+    >
+  }
+
 /* Load the model */
 export type GeneratorLLMActionLoadModel = Action & {
   __actionName: 'GENERATOR_LLM_LOAD_MODEL'
@@ -5975,6 +6203,11 @@ export type GeneratorLLMActionProcessPrompt = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'enableThinking'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'prompt'
         value?: string | DataLink | EventProperty
@@ -6037,6 +6270,11 @@ export type GeneratorLLMActionCompletion = ActionWithParams & {
         value?: string | DataLink | EventProperty
         mapping?: string
       }
+    | {
+        input: 'enableThinking'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
     | {
         input: 'prompt'
         value?: string | DataLink | EventProperty
@@ -6261,6 +6499,7 @@ Default property:
   "completionMode": "auto",
   "completionPrompt": "",
   "completionPromptTemplateType": "${}",
+  "completionEnableThinking": true,
   "completionStopWords": [
     "</s>",
     "<|end|>",
@@ -6406,6 +6645,8 @@ Default property:
               }
           schema?: {} | DataLink
         }
+    /* Enable thinking */
+    completionEnableThinking?: boolean | DataLink
     /* Stop words */
     completionStopWords?: Array<string | DataLink> | DataLink
     /* Number of tokens to predict */
@@ -6544,6 +6785,308 @@ export type GeneratorLLM = Generator &
     >
   }
 
+/* Load the model */
+export type GeneratorGGMLTTSActionLoadModel = Action & {
+  __actionName: 'GENERATOR_GGML_TTS_LOAD_MODEL'
+}
+
+/* Generate audio */
+export type GeneratorGGMLTTSActionGenerate = ActionWithParams & {
+  __actionName: 'GENERATOR_GGML_TTS_GENERATE'
+  params?: Array<{
+    input: 'text'
+    value?: string | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Clean cache */
+export type GeneratorGGMLTTSActionCleanCache = Action & {
+  __actionName: 'GENERATOR_GGML_TTS_CLEAN_CACHE'
+}
+
+/* Release context */
+export type GeneratorGGMLTTSActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_GGML_TTS_RELEASE_CONTEXT'
+}
+
+interface GeneratorGGMLTTSDef {
+  /*
+Default property:
+{
+  "vocoderUrl": "https://huggingface.co/ggml-org/WavTokenizer/resolve/main/WavTokenizer-Large-75-F16.gguf",
+  "vocoderHashType": "sha256",
+  "vocoderHash": "2356baa8631cc2995ea3465196a017a2733600d849a91180c0f97fa7fb375bbe",
+  "outputType": "play",
+  "cacheGenerated": true,
+  "autoInferEnable": false,
+  "softBreakRegex": "^[^\\r\\n\\t\\f\\v]*([\\r\\n]+|[。！？!?.]\\B)",
+  "hardBreakTime": 500,
+  "completionTemperature": 0.1,
+  "completionRepetitionPenalty": 1.1,
+  "completionTopK": 40,
+  "completionTopP": 0.9,
+  "completionMinP": 0.05,
+  "useGuideToken": false,
+  "contextSize": 4096,
+  "batchSize": 512,
+  "maxThreads": 1,
+  "accelVariant": "default",
+  "mainGpu": 0,
+  "gpuLayers": 0,
+  "useMlock": true,
+  "useMmap": true,
+  "useFlashAttn": false
+}
+  */
+  property?: {
+    /* Initialize the TTS context on generator initialization */
+    init?: boolean | DataLink
+    /* The URL or path of model  
+ We used GGUF format model, please refer to https://github.com/ggerganov/llama.cpp/tree/master#description */
+    modelUrl?: string | DataLink
+    /* Hash type of model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of model */
+    modelHash?: string | DataLink
+    /* The URL or path of vocoder model */
+    vocoderUrl?: string | DataLink
+    /* Hash type of vocoder model */
+    vocoderHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of vocoder model */
+    vocoderHash?: string | DataLink
+    /* Output mode */
+    outputType?: 'play' | 'file' | DataLink
+    /* Enable cache for generated audio */
+    cacheGenerated?: boolean | DataLink
+    /* Text to generate */
+    prompt?: string | DataLink
+    /* Speaker JSON */
+    speaker?: {} | DataLink
+    /* Auto inference when prompt changes */
+    autoInferEnable?: boolean | DataLink
+    /* Segmentation rule for auto inference */
+    softBreakRegex?: string | DataLink
+    /* Time to force inference when softBreakRegex is not satisfied */
+    hardBreakTime?: number | DataLink
+    /* Temperature */
+    completionTemperature?: number | DataLink
+    /* Repetition Penalty */
+    completionRepetitionPenalty?: number | DataLink
+    /* Top K sampling */
+    completionTopK?: number | DataLink
+    /* Top P sampling */
+    completionTopP?: number | DataLink
+    /* Min P sampling */
+    completionMinP?: number | DataLink
+    /* Set the random number generator (RNG) seed (default: -1, -1 = random seed) */
+    completionSeed?: number | DataLink
+    /* Enable guide token to help prevent hallucinations by forcing the TTS to use the correct words. */
+    useGuideToken?: boolean | DataLink
+    /* Context size, for OutTTS recommended 4096 ~ 8192 (Default to 4096) */
+    contextSize?: number | DataLink
+    /* Logical batch size for prompt processing */
+    batchSize?: number | DataLink
+    /* Number of threads */
+    maxThreads?: number | DataLink
+    /* Accelerator variant (Only for desktop)  
+ `default` - CPU / Metal (macOS)  
+ `vulkan` - Use Vulkan  
+ `cuda` - Use CUDA */
+    accelVariant?: 'default' | 'vulkan' | 'cuda' | DataLink
+    /* Main GPU index */
+    mainGpu?: number | DataLink
+    /* Number of GPU layers (NOTE: Currently not supported for Android) */
+    gpuLayers?: number | DataLink
+    /* Use memory lock */
+    useMlock?: boolean | DataLink
+    /* Use mmap */
+    useMmap?: boolean | DataLink
+    /* Use Flash Attention for inference (Recommended with GPU enabled) */
+    useFlashAttn?: boolean | DataLink
+  }
+  events?: {
+    /* Event triggered when state change */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Generated audio file */
+    generatedAudio?: () => Data
+    /* Generated audio file is playing */
+    generatedAudioPlaying?: () => Data
+  }
+}
+
+/* Local Text-to-Speech (TTS) inference based on GGML and [llama.cpp](https://github.com/ggerganov/llama.cpp)  
+ You can use any converted model on HuggingFace. */
+export type GeneratorGGMLTTS = Generator &
+  GeneratorGGMLTTSDef & {
+    templateKey: 'GENERATOR_GGML_TTS'
+    switches: Array<
+      SwitchDef &
+        GeneratorGGMLTTSDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'contextState' | 'generatedAudio' | 'generatedAudioPlaying'
+                  value: any
+                }
+          }>
+        }
+    >
+  }
+
+/* Load the model */
+export type GeneratorRerankerActionLoadModel = Action & {
+  __actionName: 'GENERATOR_RERANKER_LOAD_MODEL'
+}
+
+/* Rerank documents based on query relevance */
+export type GeneratorRerankerActionRerank = ActionWithParams & {
+  __actionName: 'GENERATOR_RERANKER_RERANK'
+  params?: Array<
+    | {
+        input: 'query'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'documents'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Release context */
+export type GeneratorRerankerActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_RERANKER_RELEASE_CONTEXT'
+}
+
+interface GeneratorRerankerDef {
+  /*
+Default property:
+{
+  "init": false,
+  "contextSize": 512,
+  "batchSize": 512,
+  "uBatchSize": 512,
+  "accelVariant": "default",
+  "mainGpu": 0,
+  "gpuLayers": 0,
+  "useMlock": true,
+  "useMmap": true,
+  "normalize": 1
+}
+  */
+  property?: {
+    /* Initialize the Reranker context on generator initialization */
+    init?: boolean | DataLink
+    /* The URL or path of reranker model (GGUF format) */
+    modelUrl?: string | DataLink
+    /* Hash of reranker model */
+    modelHash?: string | DataLink
+    /* Hash type of reranker model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Context size (0 ~ 4096) (Default to 512) */
+    contextSize?: number | DataLink
+    /* Logical batch size for processing (default: 512) */
+    batchSize?: number | DataLink
+    /* Physical maximum batch size (default: 512) */
+    uBatchSize?: number | DataLink
+    /* Accelerator variant (default: default) */
+    accelVariant?:
+      | 'default'
+      | 'avx'
+      | 'avx2'
+      | 'avx512'
+      | 'metal'
+      | 'opencl'
+      | 'vulkan'
+      | 'cuda'
+      | 'rocm'
+      | DataLink
+    /* Main GPU index (default: 0) */
+    mainGpu?: number | DataLink
+    /* Number of layers to store in VRAM (default: 0) */
+    gpuLayers?: number | DataLink
+    /* Maximum number of threads to use (default: auto) */
+    maxThreads?: number | DataLink
+    /* Use mlock to keep model in memory (default: true) */
+    useMlock?: boolean | DataLink
+    /* Use mmap for model loading (default: true) */
+    useMmap?: boolean | DataLink
+    /* Query text for reranking */
+    query?: string | DataLink
+    /* Array of documents to rerank */
+    documents?: Array<string | DataLink> | DataLink
+    /* Normalize reranking scores (default: from model config) */
+    normalize?: number | DataLink | boolean | DataLink | DataLink
+    /* Maximum number of documents to return (default: unlimited) */
+    topK?: number | DataLink
+  }
+  events?: {
+    /* Event triggered when the reranker context state changes (loading, ready, error, released) */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when an error occurs during reranker operations */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* Current state of the reranker context (loading, ready, error, released) */
+    contextState?: () => Data
+    /* Loading progress of the reranker model (0-100) */
+    contextLoadProgress?: () => Data
+    /* Detailed information about the reranker context including instance ID and processing status */
+    contextDetails?: () => Data
+    /* Result of the reranking operation containing scored and ranked documents */
+    rerankResult?: () => Data
+    /* Boolean indicating whether the reranker is currently processing a request */
+    isProcessing?: () => Data
+  }
+}
+
+/* Local rerank based on GGML and [llama.cpp](https://github.com/ggerganov/llama.cpp)  
+
+ ## Notice  
+ - The device RAM must be larger than 8GB  
+ - iOS: Supported GPU acceleration, recommended use M1+ / A17+ chip device  
+ - macOS: Supported GPU acceleration, recommended use M1+ chip device  
+ - Android: Currently not supported GPU acceleration (Coming soon), recommended use Android 13+ system  
+ - Linux / Windows: Supported GPU acceleration, currently only Vulkan backend available */
+export type GeneratorReranker = Generator &
+  GeneratorRerankerDef & {
+    templateKey: 'GENERATOR_RERANKER'
+    switches: Array<
+      SwitchDef &
+        GeneratorRerankerDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'contextLoadProgress'
+                    | 'contextDetails'
+                    | 'rerankResult'
+                    | 'isProcessing'
+                  value: any
+                }
+          }>
+        }
+    >
+  }
+
 /* Load the model */
 export type GeneratorQnnLlmActionLoadModel = Action & {
   __actionName: 'GENERATOR_QNN_LLM_LOAD_MODEL'

package/types/system.ts

@@ -467,6 +467,16 @@ export type SystemActionChannelPublish = ActionWithParams & {
   >
 }
 
+/* Delay for a certain time between actions */
+export type SystemActionDelay = ActionWithParams & {
+  __actionName: 'DELAY'
+  params?: Array<{
+    input: 'time'
+    value?: number | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
 /* [Internal] Use a shared application */
 export type SystemActionUseShareApplication = ActionWithParams & {
   __actionName: 'USE_SHARE_APPLICATION'

package/utils/data.ts

@@ -64,6 +64,7 @@ type SystemDataName =
   | 'workspaceName'
   | 'applicationInfo'
   | 'applicationName'
+  | 'deviceID'
   | 'macAddress'
   | 'bindDeviceCode'
   | 'bindDeviceCodeExpire'
@@ -343,6 +344,14 @@ export const systemDataList: Array<SystemDataInfo> = [
     type: 'string',
     value: '',
   },
+  {
+    name: 'deviceID',
+    id: 'PROPERTY_BANK_DATA_NODE_8b5c2d9e-f0a1-4b2c-8d3e-4f5a6b7c8d9e',
+    title: 'SYSTEM: Device ID',
+    description: 'Device ID of current device',
+    type: 'string',
+    value: 'unknown',
+  },
   {
     name: 'macAddress',
     id: 'PROPERTY_BANK_DATA_NODE_f01fcc78-0723-11ed-ac00-877339de1030',

package/utils/event-props.ts

@@ -761,6 +761,19 @@ export const templateEventPropsMap = {
       'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_TIME', // type: number
     ],
   },
+  GENERATOR_VAD_INFERENCE: {
+    onContextStateChange: [
+      'GENERATOR_VAD_INFERENCE_CONTEXT_STATE', // type: string
+      'GENERATOR_VAD_INFERENCE_CONTEXT_DETAILS', // type: object
+    ],
+    onError: [
+      'GENERATOR_VAD_INFERENCE_ERROR', // type: string
+    ],
+    onDetected: [
+      'GENERATOR_VAD_INFERENCE_DETECTION_SEGMENTS', // type: array
+      'GENERATOR_VAD_INFERENCE_DETECTION_TIME', // type: number
+    ],
+  },
   GENERATOR_LLM: {
     onContextStateChange: [
       'GENERATOR_LLM_CONTEXT_STATE', // type: string
@@ -791,6 +804,15 @@ export const templateEventPropsMap = {
       'GENERATOR_LLM_COMPLETION_FUNCTION_DETAILS', // type: object
     ],
   },
+  GENERATOR_GGML_TTS: {
+    onContextStateChange: [
+      'GENERATOR_GGML_TTS_CONTEXT_STATE', // type: string
+    ],
+    onError: [
+      'GENERATOR_GGML_TTS_ERROR', // type: string
+    ],
+  },
+  GENERATOR_RERANKER: {},
   GENERATOR_QNN_LLM: {
     onContextStateChange: [
       'GENERATOR_QNN_LLM_CONTEXT_STATE', // type: string