@fugood/bricks-project 2.22.0-beta.15 → 2.22.0-beta.17

package/compile/action-name-map.ts

@@ -610,6 +610,27 @@ export const templateActionNameMap = {
       realtimeVadFreqThold: 'GENERATOR_SPEECH_INFERENCE_REALTIME_VAD_FREQ_THOLD',
     },
   },
+  GENERATOR_VAD_INFERENCE: {
+    GENERATOR_VAD_INFERENCE_DETECT_FILE: {
+      fileUrl: 'GENERATOR_VAD_INFERENCE_FILE_URL',
+      threshold: 'GENERATOR_VAD_INFERENCE_THRESHOLD',
+      minSpeechDurationMs: 'GENERATOR_VAD_INFERENCE_MIN_SPEECH_DURATION_MS',
+      minSilenceDurationMs: 'GENERATOR_VAD_INFERENCE_MIN_SILENCE_DURATION_MS',
+      maxSpeechDurationS: 'GENERATOR_VAD_INFERENCE_MAX_SPEECH_DURATION_S',
+      speechPadMs: 'GENERATOR_VAD_INFERENCE_SPEECH_PAD_MS',
+      samplesOverlap: 'GENERATOR_VAD_INFERENCE_SAMPLES_OVERLAP',
+    },
+    GENERATOR_VAD_INFERENCE_DETECT_DATA: {
+      data: 'GENERATOR_VAD_INFERENCE_DATA',
+      threshold: 'GENERATOR_VAD_INFERENCE_THRESHOLD',
+      minSpeechDurationMs: 'GENERATOR_VAD_INFERENCE_MIN_SPEECH_DURATION_MS',
+      minSilenceDurationMs: 'GENERATOR_VAD_INFERENCE_MIN_SILENCE_DURATION_MS',
+      maxSpeechDurationS: 'GENERATOR_VAD_INFERENCE_MAX_SPEECH_DURATION_S',
+      speechPadMs: 'GENERATOR_VAD_INFERENCE_SPEECH_PAD_MS',
+      samplesOverlap: 'GENERATOR_VAD_INFERENCE_SAMPLES_OVERLAP',
+    },
+  },
+
   GENERATOR_LLM: {
     GENERATOR_LLM_TOKENIZE: {
       mode: 'GENERATOR_LLM_MODE',
@@ -685,6 +706,12 @@ export const templateActionNameMap = {
       text: 'GENERATOR_GGML_TTS_TEXT',
     },
   },
+  GENERATOR_RERANKER: {
+    GENERATOR_RERANKER_RERANK: {
+      query: 'GENERATOR_RERANKER_QUERY',
+      documents: 'GENERATOR_RERANKER_DOCUMENTS',
+    },
+  },
   GENERATOR_QNN_LLM: {
     GENERATOR_QNN_LLM_GENERATE: {
       prompt: 'GENERATOR_QNN_LLM_PROMPT',

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@fugood/bricks-project",
-  "version": "2.22.0-beta.15",
+  "version": "2.22.0-beta.17",
   "main": "index.ts",
   "scripts": {
     "build": "node scripts/build.js"
   },
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.11.1",
+    "@modelcontextprotocol/sdk": "^1.15.0",
     "@types/escodegen": "^0.0.10",
     "@types/lodash": "^4.17.12",
     "acorn": "^8.13.0",
@@ -14,5 +14,5 @@
     "lodash": "^4.17.4",
     "uuid": "^8.3.1"
   },
-  "gitHead": "4995b78120bd6e75df65a302398a9ba3070da3e7"
+  "gitHead": "5e8893af505fa5a7b1a902e11764c1f9ebb36a4f"
 }

package/tools/postinstall.ts

@@ -1,15 +1,7 @@
 import { $ } from 'bun'
 import { stat, readFile, writeFile } from 'fs/promises'
-const cwd = process.cwd()
-
-const libFiles = ['types', 'utils', 'index.ts']
 
-await $`mkdir -p ${cwd}/project`
-for (const file of libFiles) {
-  await $`cp -r ${__dirname}/../${file} ${cwd}/project`
-}
-
-console.log('Copied files to project/')
+const cwd = process.cwd()
 
 async function exists(f: string) {
   try {
@@ -20,6 +12,21 @@ async function exists(f: string) {
   }
 }
 
+// handle flag --skip-copy
+const skipCopyProject = process.argv.includes('--skip-copy-project')
+if (skipCopyProject) {
+  console.log('Skipping copy of files to project/')
+} else {
+
+  const libFiles = ['types', 'utils', 'index.ts']
+  
+  await $`mkdir -p ${cwd}/project`
+  for (const file of libFiles) {
+    await $`cp -r ${__dirname}/../${file} ${cwd}/project`
+  }
+  console.log('Copied files to project/')
+}
+
 const projectMcpServer = {
   command: 'bun',
   args: [`${cwd}/node_modules/@fugood/bricks-project/tools/mcp-server.ts`],

package/types/common.ts

@@ -39,7 +39,7 @@ export type SubpsaceAction = string
 export type Action = {
   __actionName: string
   parent: 'Brick' | 'Generator' | 'Subspace' | 'System'
-  name: string
+  name?: string
 }
 
 // Find correct key in bricks-project/utils/event-props for EventAction
@@ -65,7 +65,7 @@ export type ItemBrickID = string
 
 export type EventAction = {
   handler: 'system' | (() => Brick | Generator) | SubspaceID | ItemBrickID
-  action: Action
+  action: ActionWithParams | ActionWithDataParams
   waitAsync?: boolean
 }
 

package/types/generators.ts

@@ -4856,6 +4856,7 @@ Default property:
   "doSample": true,
   "outputType": "play",
   "cacheGenerated": true,
+  "speed": 1,
   "autoInferEnable": false,
   "softBreakRegex": "^[^\\r\\n\\t\\f\\v]*([\\r\\n]+|[。！？!?.]\\B)",
   "hardBreakTime": 500,
@@ -4867,30 +4868,9 @@ Default property:
     init?: boolean | DataLink
     /* TTS model  
  The mms-tts models are licensed under CC-BY-NC-4.0 */
-    model?:
-      | 'Custom'
-      | 'BricksDisplay/vits-eng'
-      | 'BricksDisplay/vits-cmn'
-      | 'BricksDisplay/ellie-Bert-VITS2'
-      | 'onnx-community/OuteTTS-1.0-0.6B-ONNX'
-      | 'mms-tts-ara (NC)'
-      | 'mms-tts-deu (NC)'
-      | 'mms-tts-eng (NC)'
-      | 'mms-tts-fra (NC)'
-      | 'mms-tts-hin (NC)'
-      | 'mms-tts-kor (NC)'
-      | 'mms-tts-por (NC)'
-      | 'mms-tts-ron (NC)'
-      | 'mms-tts-rus (NC)'
-      | 'mms-tts-spa (NC)'
-      | 'mms-tts-vie (NC)'
-      | 'mms-tts-yor (NC)'
-      | 'speecht5_tts'
-      | DataLink
+    model?: string | DataLink
     /* Model type */
-    modelType?: 'auto' | 'vits' | 'bert_vits2' | 'speecht5' | 'outetts-1.0' | DataLink
-    /* Load quantized model (deprecated, use `quantizeType` instead) */
-    quantized?: boolean | DataLink
+    modelType?: string | DataLink
     /* Quantize type */
     quantizeType?:
       | 'auto'
@@ -4903,18 +4883,17 @@ Default property:
       | 'bnb4'
       | 'q4f16'
       | DataLink
-    /* Custom model name  
- Choose model from https://huggingface.co/models?pipeline_tag=text-to-audio&library=transformers.js */
-    customModel?: string | DataLink
     /* Vocoder model for SpeechT5 */
     vocoderModel?: 'Custom' | 'speecht5_hifigan' | DataLink
     /* Custom vocoder model  
  Choose model from https://huggingface.co/models?library=transformers.js&other=hifigan */
     customVocoderModel?: string | DataLink
-    /* XVector speaker embedding for HiFi-GAN */
+    /* Speaker embedding, for SpeechT5 or StyleTTS (Kokoro) */
     speakerEmbedUrl?: string | DataLink
-    /* MD5 checksum of `speakerEmbedUrl` */
-    speakerEmbedMd5?: string | DataLink
+    /* Hash of `speakerEmbedUrl` */
+    speakerEmbedHash?: string | DataLink
+    /* Hash type of `speakerEmbedUrl` */
+    speakerEmbedHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
     /* Speaker config, for OuteTTS model */
     speakerConfig?: {} | DataLink
     /* Audio token generation max length */
@@ -4929,6 +4908,8 @@ Default property:
     outputType?: 'play' | 'file' | DataLink
     /* Enable cache for generated audio */
     cacheGenerated?: boolean | DataLink
+    /* Speed of the generated audio, for StyleTTS (Kokoro) */
+    speed?: number | DataLink
     /* Text to generate */
     prompt?: string | DataLink
     /* Auto inference when prompt changes */
@@ -5042,7 +5023,6 @@ interface GeneratorOnnxLLMDef {
   /*
 Default property:
 {
-  "model": "BricksDisplay/phi-1_5-q4",
   "modelType": "auto",
   "toolCallParser": "llama3_json",
   "toolChoice": "auto",
@@ -5061,70 +5041,9 @@ Default property:
     /* Initialize the TTS context on generator initialization */
     init?: boolean | DataLink
     /* LLM model */
-    model?:
-      | 'Custom'
-      | 'onnx-community/gemma-3-1b-it-ONNX'
-      | 'BricksDisplay/phi-1_5'
-      | 'BricksDisplay/phi-1_5-q4'
-      | 'onnx-community/Phi-3.5-vision-instruct'
-      | 'onnx-community/Phi-3-vision-128k-instruct'
-      | 'onnx-community/Phi-4-mini-instruct-ONNX-MHA'
-      | 'onnx-community/Qwen2.5-0.5B'
-      | 'onnx-community/Qwen2.5-0.5B-Instruct'
-      | 'onnx-community/Qwen2.5-1.5B'
-      | 'onnx-community/Qwen2.5-1.5B-Instruct'
-      | 'onnx-community/Qwen2-VL-2B-Instruct'
-      | 'stablelm-2-1_6b'
-      | 'BricksDisplay/stablelm-2-1_6b-q4'
-      | 'stablelm-2-zephyr-1_6b'
-      | 'BricksDisplay/stablelm-2-zephyr-1_6b-q4'
-      | 'BricksDisplay/Llama-2-7b-chat-q4'
-      | 'TinyLLama-v0'
-      | 'TinyLlama-1.1B-Chat-v1.0'
-      | 'BricksDisplay/TinyLlama-1.1B-Chat-v1.0-q4'
-      | 'llama-160m'
-      | 'llama-68m'
-      | 'BricksDisplay/Yi-6B-q4'
-      | 'BricksDisplay/Yi-6B-Chat-q4'
-      | 'BricksDisplay/Mistral-7B-v0.1-q4'
-      | 'BricksDisplay/Mistral-7B-Instruct-v0.2-q4'
-      | 'BricksDisplay/Breeze-7B-Base-v1_0-q4'
-      | 'BricksDisplay/Breeze-7B-Instruct-v1_0-q4'
-      | 'gpt2'
-      | 'distilgpt2'
-      | 'gpt-neo-125M'
-      | 'opt-125m'
-      | 'opt-350m'
-      | 'bloom-560m'
-      | 'bloomz-560m'
-      | 't5-small'
-      | 't5-base'
-      | 'flan-t5-small'
-      | 'flan-t5-base'
-      | 'mt5-small'
-      | 'mt5-base'
-      | 'long-t5-lobal-base'
-      | 'long-t5-tglobal-base'
-      | DataLink
+    model?: string | DataLink
     /* Model type */
-    modelType?:
-      | 'auto'
-      | 'text-generation'
-      | 'qwen2-vl'
-      | 'paligemma'
-      | 'llava'
-      | 'llava_onevision'
-      | 'moondream1'
-      | 'florence2'
-      | 'idefics3'
-      | 'smolvlm'
-      | 'phi3_v'
-      | 't5'
-      | 'mt5'
-      | 'longt5'
-      | DataLink
-    /* Load quantized model (deprecated, use `quantizeType` instead) */
-    quantized?: boolean | DataLink
+    modelType?: string | DataLink
     /* Quantize type */
     quantizeType?:
       | 'auto'
@@ -5137,10 +5056,6 @@ Default property:
       | 'bnb4'
       | 'q4f16'
       | DataLink
-    /* Custom model name  
- Choose model from https://huggingface.co/models?pipeline_tag=text2text-generation&library=transformers.js  
- or https://huggingface.co/models?pipeline_tag=text-generation&library=transformers.js&sort=trending */
-    customModel?: string | DataLink
     /* Prompt to inference */
     prompt?: string | DataLink
     /* Messages to inference */
@@ -5270,27 +5185,9 @@ Default property:
     /* Initialize the TTS context on generator initialization */
     init?: boolean | DataLink
     /* STT model */
-    model?:
-      | 'Custom'
-      | 'whisper-tiny'
-      | 'whisper-tiny.en'
-      | 'whisper-small'
-      | 'whisper-small.en'
-      | 'whisper-base'
-      | 'whisper-base.en'
-      | 'whisper-medium'
-      | 'whisper-medium.en'
-      | 'whisper-large'
-      | 'whisper-large-v2'
-      | 'whisper-large-v3'
-      | 'mms-1b-all'
-      | 'mms-1b-fl102'
-      | 'mms-1b-l1107'
-      | DataLink
+    model?: string | DataLink
     /* Model type */
-    modelType?: 'auto' | 'whisper' | 'hubert' | 'wav2vec2' | 'wav2vec2-bert' | DataLink
-    /* Load quantized model (deprecated, use `quantizeType` instead) */
-    quantized?: boolean | DataLink
+    modelType?: string | DataLink
     /* Quantize type */
     quantizeType?:
       | 'auto'
@@ -5303,9 +5200,6 @@ Default property:
       | 'bnb4'
       | 'q4f16'
       | DataLink
-    /* Custom model name  
- Choose model from https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&library=transformers.js */
-    customModel?: string | DataLink
     /* Return timestamps */
     returnTimestamps?: 'none' | 'enable' | 'word' | DataLink
     /* Transcription language  
@@ -5538,7 +5432,7 @@ export type GeneratorSpeechInferenceActionTranscribeData = ActionWithParams & {
   >
 }
 
-/* Transcribe microphone audio source */
+/* [Deprecated] Transcribe microphone audio source */
 export type GeneratorSpeechInferenceActionTranscribeRealtime = ActionWithParams & {
   __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_REALTIME'
   params?: Array<
@@ -5605,7 +5499,7 @@ export type GeneratorSpeechInferenceActionTranscribeRealtime = ActionWithParams
   >
 }
 
-/* Stop transcribing microphone audio source */
+/* [Deprecated] Stop transcribing microphone audio source */
 export type GeneratorSpeechInferenceActionTranscribeRealtimeStop = Action & {
   __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_REALTIME_STOP'
 }
@@ -5853,7 +5747,7 @@ Default property:
     inferRealtimeVadFreqThold?: number | DataLink
   }
   events?: {
-    /* Event triggered when load is done */
+    /* Event triggered when context state changes */
     onContextStateChange?: Array<EventAction>
     /* Event triggered when error occurs */
     onError?: Array<EventAction>
@@ -5909,6 +5803,369 @@ export type GeneratorSpeechInference = Generator &
     >
   }
 
+/* Load the model */
+export type GeneratorVadInferenceActionLoadModel = Action & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_LOAD_MODEL'
+}
+
+/* Detect speech in audio file. You can provide `File URL` property, if not provided, it will use the default `File URL` */
+export type GeneratorVadInferenceActionDetectFile = ActionWithParams & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_DETECT_FILE'
+  params?: Array<
+    | {
+        input: 'fileUrl'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSpeechDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSilenceDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxSpeechDurationS'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'speechPadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'samplesOverlap'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Detect speech in audio data. Currently only support base64 encoded audio data (16-bit PCM, mono, 16kHz) */
+export type GeneratorVadInferenceActionDetectData = ActionWithParams & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_DETECT_DATA'
+  params?: Array<
+    | {
+        input: 'data'
+        value?: any | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSpeechDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSilenceDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxSpeechDurationS'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'speechPadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'samplesOverlap'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Clear downloaded files (model, audio) & current jobs */
+export type GeneratorVadInferenceActionClearDownload = Action & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_CLEAR_DOWNLOAD'
+}
+
+/* Release context */
+export type GeneratorVadInferenceActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_RELEASE_CONTEXT'
+}
+
+interface GeneratorVadInferenceDef {
+  /*
+Default property:
+{
+  "init": false,
+  "modelName": "silero-v5.1.2",
+  "modelUseGPU": true,
+  "modelThreads": 4,
+  "detectThreshold": 0.5,
+  "detectMinSpeechDurationMs": 250,
+  "detectMinSilenceDurationMs": 100,
+  "detectMaxSpeechDurationS": 30,
+  "detectSpeechPadMs": 30,
+  "detectSamplesOverlap": 0.1
+}
+  */
+  property?: {
+    /* Initialize the VAD context on generator initialization  
+ Please note that it will take some RAM depending on the model size */
+    init?: boolean | DataLink
+    /* Use model name, currently only supports the Silero VAD model.  
+ The model download progress will be done in preload stage or  the generator initialization stage.  
+ You can also choose `custom` option and set `Model URL` and `Model SHA1` to use your own model */
+    modelName?: 'custom' | 'silero-v5.1.2' | DataLink
+    /* The URL or path of model  
+ We used `ggml` format model, please refer to https://huggingface.co/ggml-org/whisper-vad */
+    modelUrl?: string | DataLink
+    /* Hash type of model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of model */
+    modelHash?: string | DataLink
+    /* Use GPU Acceleration for inference. Currently iOS only. */
+    modelUseGPU?: boolean | DataLink
+    /* Number of threads to use for processing */
+    modelThreads?: number | DataLink
+    /* Speech probability threshold (0.0-1.0) */
+    detectThreshold?: number | DataLink
+    /* Minimum speech duration in milliseconds */
+    detectMinSpeechDurationMs?: number | DataLink
+    /* Minimum silence duration in milliseconds */
+    detectMinSilenceDurationMs?: number | DataLink
+    /* Maximum speech duration in seconds */
+    detectMaxSpeechDurationS?: number | DataLink
+    /* Padding around speech segments in milliseconds */
+    detectSpeechPadMs?: number | DataLink
+    /* Overlap between analysis windows (0.0-1.0) */
+    detectSamplesOverlap?: number | DataLink
+    /* The file URL or path to be analyzed.  
+ It only supported `wav` format with 16kHz sample rate & single (mono) channel */
+    detectFileUrl?: string | DataLink
+    /* MD5 of file to be analyzed */
+    detectFileMd5?: string | DataLink
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+    /* Event triggered when got detection result */
+    onDetected?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Context details */
+    contextDetails?: () => Data
+    /* Is detecting */
+    isDetecting?: () => Data
+    /* Detection segments result */
+    detectionSegments?: () => Data
+    /* Detection details */
+    detectionDetails?: () => Data
+  }
+}
+
+/* Local Voice Activity Detection (VAD) inference based on GGML and [whisper.rn](https://github.com/mybigday/whisper.rn) */
+export type GeneratorVadInference = Generator &
+  GeneratorVadInferenceDef & {
+    templateKey: 'GENERATOR_VAD_INFERENCE'
+    switches: Array<
+      SwitchDef &
+        GeneratorVadInferenceDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'contextDetails'
+                    | 'isDetecting'
+                    | 'detectionSegments'
+                    | 'detectionDetails'
+                  value: any
+                }
+          }>
+        }
+    >
+  }
+
+/* Start realtime transcription */
+export type GeneratorRealtimeTranscriptionActionStart = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_START'
+}
+
+/* Stop realtime transcription */
+export type GeneratorRealtimeTranscriptionActionStop = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_STOP'
+}
+
+/* Force move to next slice */
+export type GeneratorRealtimeTranscriptionActionNextSlice = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_NEXT_SLICE'
+}
+
+/* Reset transcriber state */
+export type GeneratorRealtimeTranscriptionActionReset = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_RESET'
+}
+
+interface GeneratorRealtimeTranscriptionDef {
+  /*
+Default property:
+{
+  "sttLivePolicy": "only-in-use",
+  "vadInferenceLivePolicy": "only-in-use",
+  "vadEnabled": true,
+  "audioSliceSec": 30,
+  "audioMinSec": 1,
+  "maxSlicesInMemory": 5,
+  "vadStrategy": "use-preset",
+  "vadPreset": "default",
+  "autoSliceOnSpeechEnd": true,
+  "autoSliceThreshold": 2,
+  "initialPrompt": "",
+  "promptPreviousSlices": false,
+  "saveAudio": true,
+  "testMode": false,
+  "testPlaybackSpeed": 1,
+  "testChunkDurationMs": 100,
+  "testLoop": false
+}
+  */
+  property?: {
+    /* STT Generator for Whisper context */
+    sttGeneratorId?: string | DataLink
+    /* STT Live Policy. If the policy is `only-in-use`, the STT context will be released when not in use. */
+    sttLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* VAD Inference Generator for voice activity detection */
+    vadInferenceGeneratorId?: string | DataLink
+    /* VAD Inference Live Policy. If the policy is `only-in-use`, the VAD Inference context will be released when not in use. */
+    vadInferenceLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* Enable VAD (Voice Activity Detection) */
+    vadEnabled?: boolean | DataLink
+    /* Audio slice duration in seconds */
+    audioSliceSec?: number | DataLink
+    /* Minimum audio duration to start transcription in seconds */
+    audioMinSec?: number | DataLink
+    /* Maximum number of slices to keep in memory */
+    maxSlicesInMemory?: number | DataLink
+    /* VAD Strategy */
+    vadStrategy?: 'use-preset' | 'use-generator-options' | DataLink
+    /* VAD preset configuration */
+    vadPreset?:
+      | 'default'
+      | 'sensitive'
+      | 'very-sensitive'
+      | 'conservative'
+      | 'very-conservative'
+      | 'continuous-speech'
+      | 'meeting'
+      | 'noisy-environment'
+      | DataLink
+    /* Auto slice on speech end */
+    autoSliceOnSpeechEnd?: boolean | DataLink
+    /* Auto slice threshold in seconds */
+    autoSliceThreshold?: number | DataLink
+    /* Initial prompt for transcription */
+    initialPrompt?: string | DataLink
+    /* Include previous slices in prompt */
+    promptPreviousSlices?: boolean | DataLink
+    /* Enable audio output saving (auto-generates file path) */
+    saveAudio?: boolean | DataLink
+    /* Use test mode with file simulation */
+    testMode?: boolean | DataLink
+    /* Test audio file path for simulation */
+    testFilePath?: string | DataLink
+    /* Test audio file hash */
+    testFileHash?: string | DataLink
+    /* Test audio file hash type */
+    testFileHashType?: string | DataLink
+    /* Test playback speed */
+    testPlaybackSpeed?: number | DataLink
+    /* Test chunk duration in milliseconds */
+    testChunkDurationMs?: number | DataLink
+    /* Loop test audio file */
+    testLoop?: boolean | DataLink
+  }
+  events?: {
+    /* Event triggered when transcription starts, processes, or ends */
+    onTranscribe?: Array<EventAction>
+    /* Event triggered on VAD (Voice Activity Detection) events */
+    onVad?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+    /* Event triggered when status changes */
+    onStatusChange?: Array<EventAction>
+    /* Event triggered when statistics update */
+    onStatsUpdate?: Array<EventAction>
+    /* Event triggered when transcription ends */
+    onEnd?: Array<EventAction>
+  }
+  outlets?: {
+    /* Is realtime transcription currently active */
+    isActive?: () => Data
+    /* Is currently transcribing audio */
+    isTranscribing?: () => Data
+    /* Current transcription results */
+    results?: () => Data
+    /* Current transcription result text */
+    resultText?: () => Data
+    /* Current statistics */
+    statistics?: () => Data
+    /* Latest transcribe event */
+    lastTranscribeEvent?: () => Data
+    /* Latest VAD event */
+    lastVadEvent?: () => Data
+    /* Audio output file path (auto-generated when saving audio) */
+    audioOutputPath?: () => Data
+  }
+}
+
+/* Realtime speech-to-text transcription using Whisper and VAD with live audio streaming */
+export type GeneratorRealtimeTranscription = Generator &
+  GeneratorRealtimeTranscriptionDef & {
+    templateKey: 'GENERATOR_REALTIME_TRANSCRIPTION'
+    switches: Array<
+      SwitchDef &
+        GeneratorRealtimeTranscriptionDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'isActive'
+                    | 'isTranscribing'
+                    | 'results'
+                    | 'resultText'
+                    | 'statistics'
+                    | 'lastTranscribeEvent'
+                    | 'lastVadEvent'
+                    | 'audioOutputPath'
+                  value: any
+                }
+          }>
+        }
+    >
+  }
+
 /* Load the model */
 export type GeneratorLLMActionLoadModel = Action & {
   __actionName: 'GENERATOR_LLM_LOAD_MODEL'
@@ -6492,7 +6749,7 @@ Default property:
     completionIgnoreEOS?: boolean | DataLink
   }
   events?: {
-    /* Event triggered when load is done */
+    /* Event triggered when context state changes */
     onContextStateChange?: Array<EventAction>
     /* Event triggered when error occurs */
     onError?: Array<EventAction>
@@ -6731,6 +6988,149 @@ export type GeneratorGGMLTTS = Generator &
     >
   }
 
+/* Load the model */
+export type GeneratorRerankerActionLoadModel = Action & {
+  __actionName: 'GENERATOR_RERANKER_LOAD_MODEL'
+}
+
+/* Rerank documents based on query relevance */
+export type GeneratorRerankerActionRerank = ActionWithParams & {
+  __actionName: 'GENERATOR_RERANKER_RERANK'
+  params?: Array<
+    | {
+        input: 'query'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'documents'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Release context */
+export type GeneratorRerankerActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_RERANKER_RELEASE_CONTEXT'
+}
+
+interface GeneratorRerankerDef {
+  /*
+Default property:
+{
+  "init": false,
+  "contextSize": 512,
+  "batchSize": 512,
+  "uBatchSize": 512,
+  "accelVariant": "default",
+  "mainGpu": 0,
+  "gpuLayers": 0,
+  "useMlock": true,
+  "useMmap": true,
+  "normalize": 1
+}
+  */
+  property?: {
+    /* Initialize the Reranker context on generator initialization */
+    init?: boolean | DataLink
+    /* The URL or path of reranker model (GGUF format) */
+    modelUrl?: string | DataLink
+    /* Hash of reranker model */
+    modelHash?: string | DataLink
+    /* Hash type of reranker model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Context size (0 ~ 4096) (Default to 512) */
+    contextSize?: number | DataLink
+    /* Logical batch size for processing (default: 512) */
+    batchSize?: number | DataLink
+    /* Physical maximum batch size (default: 512) */
+    uBatchSize?: number | DataLink
+    /* Accelerator variant (default: default) */
+    accelVariant?:
+      | 'default'
+      | 'avx'
+      | 'avx2'
+      | 'avx512'
+      | 'metal'
+      | 'opencl'
+      | 'vulkan'
+      | 'cuda'
+      | 'rocm'
+      | DataLink
+    /* Main GPU index (default: 0) */
+    mainGpu?: number | DataLink
+    /* Number of layers to store in VRAM (default: 0) */
+    gpuLayers?: number | DataLink
+    /* Maximum number of threads to use (default: auto) */
+    maxThreads?: number | DataLink
+    /* Use mlock to keep model in memory (default: true) */
+    useMlock?: boolean | DataLink
+    /* Use mmap for model loading (default: true) */
+    useMmap?: boolean | DataLink
+    /* Query text for reranking */
+    query?: string | DataLink
+    /* Array of documents to rerank */
+    documents?: Array<string | DataLink> | DataLink
+    /* Normalize reranking scores (default: from model config) */
+    normalize?: number | DataLink | boolean | DataLink | DataLink
+    /* Maximum number of documents to return (default: unlimited) */
+    topK?: number | DataLink
+  }
+  events?: {
+    /* Event triggered when the reranker context state changes (loading, ready, error, released) */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when an error occurs during reranker operations */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* Current state of the reranker context (loading, ready, error, released) */
+    contextState?: () => Data
+    /* Loading progress of the reranker model (0-100) */
+    contextLoadProgress?: () => Data
+    /* Detailed information about the reranker context including instance ID and processing status */
+    contextDetails?: () => Data
+    /* Result of the reranking operation containing scored and ranked documents */
+    rerankResult?: () => Data
+    /* Boolean indicating whether the reranker is currently processing a request */
+    isProcessing?: () => Data
+  }
+}
+
+/* Local rerank based on GGML and [llama.cpp](https://github.com/ggerganov/llama.cpp)  
+
+ ## Notice  
+ - The device RAM must be larger than 8GB  
+ - iOS: Supported GPU acceleration, recommended use M1+ / A17+ chip device  
+ - macOS: Supported GPU acceleration, recommended use M1+ chip device  
+ - Android: Currently not supported GPU acceleration (Coming soon), recommended use Android 13+ system  
+ - Linux / Windows: Supported GPU acceleration, currently only Vulkan backend available */
+export type GeneratorReranker = Generator &
+  GeneratorRerankerDef & {
+    templateKey: 'GENERATOR_RERANKER'
+    switches: Array<
+      SwitchDef &
+        GeneratorRerankerDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'contextLoadProgress'
+                    | 'contextDetails'
+                    | 'rerankResult'
+                    | 'isProcessing'
+                  value: any
+                }
+          }>
+        }
+    >
+  }
+
 /* Load the model */
 export type GeneratorQnnLlmActionLoadModel = Action & {
   __actionName: 'GENERATOR_QNN_LLM_LOAD_MODEL'
@@ -6858,7 +7258,7 @@ Default property:
     greedy?: boolean | DataLink
   }
   events?: {
-    /* Event triggered when load is done */
+    /* Event triggered when context state changes */
     onContextStateChange?: Array<EventAction>
     /* Event triggered when generate is done */
     onGenerate?: Array<EventAction>

package/utils/event-props.ts

@@ -761,6 +761,46 @@ export const templateEventPropsMap = {
       'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_TIME', // type: number
     ],
   },
+  GENERATOR_VAD_INFERENCE: {
+    onContextStateChange: [
+      'GENERATOR_VAD_INFERENCE_CONTEXT_STATE', // type: string
+      'GENERATOR_VAD_INFERENCE_CONTEXT_DETAILS', // type: object
+    ],
+    onError: [
+      'GENERATOR_VAD_INFERENCE_ERROR', // type: string
+    ],
+    onDetected: [
+      'GENERATOR_VAD_INFERENCE_DETECTION_SEGMENTS', // type: array
+      'GENERATOR_VAD_INFERENCE_DETECTION_TIME', // type: number
+    ],
+  },
+  GENERATOR_REALTIME_TRANSCRIPTION: {
+    onTranscribe: [
+      'GENERATOR_REALTIME_TRANSCRIPTION_TRANSCRIBE_EVENT', // type: object
+    ],
+    onVad: [
+      'GENERATOR_REALTIME_TRANSCRIPTION_VAD_EVENT', // type: object
+      'GENERATOR_REALTIME_TRANSCRIPTION_VAD_EVENT_TYPE', // type: string
+      'GENERATOR_REALTIME_TRANSCRIPTION_VAD_EVENT_CONFIDENCE', // type: number
+      'GENERATOR_REALTIME_TRANSCRIPTION_VAD_EVENT_DURATION', // type: number
+      'GENERATOR_REALTIME_TRANSCRIPTION_VAD_EVENT_SLICE_INDEX', // type: number
+    ],
+    onError: [
+      'GENERATOR_REALTIME_TRANSCRIPTION_ERROR', // type: string
+    ],
+    onStatusChange: [
+      'GENERATOR_REALTIME_TRANSCRIPTION_IS_ACTIVE', // type: bool
+    ],
+    onStatsUpdate: [
+      'GENERATOR_REALTIME_TRANSCRIPTION_STATS_TYPE', // type: string
+      'GENERATOR_REALTIME_TRANSCRIPTION_STATS_TIMESTAMP', // type: number
+      'GENERATOR_REALTIME_TRANSCRIPTION_STATS', // type: object
+    ],
+    onEnd: [
+      'GENERATOR_REALTIME_TRANSCRIPTION_END_RESULTS', // type: array
+      'GENERATOR_REALTIME_TRANSCRIPTION_END_AUDIO_OUTPUT_PATH', // type: string
+    ],
+  },
   GENERATOR_LLM: {
     onContextStateChange: [
       'GENERATOR_LLM_CONTEXT_STATE', // type: string
@@ -799,6 +839,7 @@ export const templateEventPropsMap = {
       'GENERATOR_GGML_TTS_ERROR', // type: string
     ],
   },
+  GENERATOR_RERANKER: {},
   GENERATOR_QNN_LLM: {
     onContextStateChange: [
       'GENERATOR_QNN_LLM_CONTEXT_STATE', // type: string