@fugood/bricks-project 2.22.0-beta.9 → 2.22.0

package/types/generators/SpeechToTextGgml.ts

@@ -0,0 +1,462 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Load the model */
+export type GeneratorSpeechInferenceActionLoadModel = Action & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_LOAD_MODEL'
+}
+
+/* Transcribe audio file. You can provide `File URL` property, if not provided, it will use the default `File URL` */
+export type GeneratorSpeechInferenceActionTranscribeFile = ActionWithParams & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_FILE'
+  params?: Array<
+    | {
+        input: 'fileUrl'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'beamSize'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'language'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'translate'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Transcribe audio data. Currently only support base64 encoded audio data (16-bit PCM, mono, 16kHz) */
+export type GeneratorSpeechInferenceActionTranscribeData = ActionWithParams & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_DATA'
+  params?: Array<
+    | {
+        input: 'data'
+        value?: any | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'beamSize'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'language'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'translate'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* [Deprecated] Transcribe microphone audio source */
+export type GeneratorSpeechInferenceActionTranscribeRealtime = ActionWithParams & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_REALTIME'
+  params?: Array<
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'beamSize'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'language'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'translate'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'realtimeAudioSeconds'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'realtimeAudioSliceSeconds'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'realtimeAudioMinSeconds'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'realtimeSaveAudio'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'realtimeVadEnabled'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'realtimeVadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'realtimeVadThold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'realtimeVadFreqThold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* [Deprecated] Stop transcribing microphone audio source */
+export type GeneratorSpeechInferenceActionTranscribeRealtimeStop = Action & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_REALTIME_STOP'
+}
+
+/* Clear downloaded files (model, audio) & current jobs */
+export type GeneratorSpeechInferenceActionClearDownload = Action & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_CLEAR_DOWNLOAD'
+}
+
+/* Release context */
+export type GeneratorSpeechInferenceActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_RELEASE_CONTEXT'
+}
+
+interface GeneratorSpeechInferenceDef {
+  /*
+Default property:
+{
+  "init": false,
+  "accelVariant": "default",
+  "modelName": "base-q8_0",
+  "modelUseCoreML": false,
+  "modelUseGPU": true,
+  "modelUseFlashAttn": false,
+  "inferLanguage": "Auto",
+  "inferRealtimeAudioSeconds": 30,
+  "inferRealtimeAudioSliceSeconds": 30,
+  "inferRealtimeSaveAudio": false,
+  "inferRealtimeVadEnabled": false,
+  "inferRealtimeVadMs": 2000,
+  "inferRealtimeVadThold": 0.6,
+  "inferRealtimeVadFreqThold": 100
+}
+  */
+  property?: {
+    /* Initialize the Whisper context on generator initialization  
+ Please note that it will take some RAM depending on the model size */
+    init?: boolean | DataLink
+    /* Accelerator variant (Only for desktop)  
+ `default` - CPU / Metal (macOS)  
+ `vulkan` - Use Vulkan  
+ `cuda` - Use CUDA */
+    accelVariant?: 'default' | 'vulkan' | 'cuda' | DataLink
+    /* Use model name,  the model download progress will be done in preload stage or  the generator initialization stage.  
+ We used `ggml` format model, please refer to https://huggingface.co/BricksDisplay/whisper-ggml  
+ You can also choose `custom` option and set `Model URL` and `Model MD5` to use your own model */
+    modelName?:
+      | 'custom'
+      | 'tiny'
+      | 'tiny-q5_1'
+      | 'tiny-q8_0'
+      | 'tiny.en'
+      | 'tiny.en-q5_1'
+      | 'tiny.en-q8_0'
+      | 'base'
+      | 'base-q5_1'
+      | 'base-q8_0'
+      | 'base.en'
+      | 'base.en-q5_1'
+      | 'base.en-q8_0'
+      | 'small'
+      | 'small-q5_1'
+      | 'small-q8_0'
+      | 'small.en'
+      | 'small.en-q5_1'
+      | 'small.en-q8_0'
+      | 'medium'
+      | 'medium-q5_0'
+      | 'medium-q8_0'
+      | 'medium.en'
+      | 'medium.en-q5_1'
+      | 'medium.en-q8_0'
+      | 'large-v1'
+      | 'large-v2'
+      | 'large-v2-q5_0'
+      | 'large-v2-q8_0'
+      | 'large-v3'
+      | 'large-v3-q5_0'
+      | 'large-v3-q8_0'
+      | 'large-v3-turbo'
+      | 'large-v3-turbo-q5_0'
+      | 'large-v3-turbo-q8_0'
+      | 'small.en-tdrz'
+      | 'small.en-tdrz-q5_0'
+      | 'small.en-tdrz-q8_0'
+      | 'distil-small.en'
+      | 'distil-small.en-q5_0'
+      | 'distil-small.en-q8_0'
+      | 'distil-medium.en'
+      | 'distil-medium.en-q5_0'
+      | 'distil-medium.en-q8_0'
+      | 'distil-large-v3'
+      | 'distil-large-v3-q5_0'
+      | 'distil-large-v3-q8_0'
+      | DataLink
+    /* The URL or path of model  
+ We used `ggml` format model, please refer to https://github.com/ggerganov/whisper.cpp/tree/master/models */
+    modelUrl?: string | DataLink
+    /* Hash type of model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of model */
+    modelHash?: string | DataLink
+    /* [Unstable] iOS: Use CoreML model for inference */
+    modelUseCoreML?: boolean | DataLink
+    /* Use GPU Acceleration for inference. Currently iOS only, if it's enabled, Core ML option will be ignored. */
+    modelUseGPU?: boolean | DataLink
+    /* Use Flash Attention for inference (Recommended with GPU enabled) */
+    modelUseFlashAttn?: boolean | DataLink
+    /* The language of the file to be inferred */
+    inferLanguage?:
+      | 'Auto'
+      | 'English (en)'
+      | 'Chinese (zh)'
+      | 'German (de)'
+      | 'Spanish (es)'
+      | 'Russian (ru)'
+      | 'Korean (ko)'
+      | 'French (fr)'
+      | 'Japanese (ja)'
+      | 'Portuguese (pt)'
+      | 'Turkish (tr)'
+      | 'Polish (pl)'
+      | 'Catalan (ca)'
+      | 'Dutch (nl)'
+      | 'Arabic (ar)'
+      | 'Swedish (sv)'
+      | 'Italian (it)'
+      | 'Indonesian (id)'
+      | 'Hindi (hi)'
+      | 'Finnish (fi)'
+      | 'Vietnamese (vi)'
+      | 'Hebrew (he)'
+      | 'Ukrainian (uk)'
+      | 'Greek (el)'
+      | 'Malay (ms)'
+      | 'Czech (cs)'
+      | 'Romanian (ro)'
+      | 'Danish (da)'
+      | 'Hungarian (hu)'
+      | 'Tamil (ta)'
+      | 'Norwegian (no)'
+      | 'Thai (th)'
+      | 'Urdu (ur)'
+      | 'Croatian (hr)'
+      | 'Bulgarian (bg)'
+      | 'Lithuanian (lt)'
+      | 'Latin (la)'
+      | 'Maori (mi)'
+      | 'Malayalam (ml)'
+      | 'Welsh (cy)'
+      | 'Slovak (sk)'
+      | 'Telugu (te)'
+      | 'Persian (fa)'
+      | 'Latvian (lv)'
+      | 'Bengali (bn)'
+      | 'Serbian (sr)'
+      | 'Azerbaijani (az)'
+      | 'Slovenian (sl)'
+      | 'Kannada (kn)'
+      | 'Estonian (et)'
+      | 'Macedonian (mk)'
+      | 'Breton (br)'
+      | 'Basque (eu)'
+      | 'Icelandic (is)'
+      | 'Armenian (hy)'
+      | 'Nepali (ne)'
+      | 'Mongolian (mn)'
+      | 'Bosnian (bs)'
+      | 'Kazakh (kk)'
+      | 'Albanian (sq)'
+      | 'Swahili (sw)'
+      | 'Galician (gl)'
+      | 'Marathi (mr)'
+      | 'Punjabi (pa)'
+      | 'Sinhala (si)'
+      | 'Khmer (km)'
+      | 'Shona (sn)'
+      | 'Yoruba (yo)'
+      | 'Somali (so)'
+      | 'Afrikaans (af)'
+      | 'Occitan (oc)'
+      | 'Georgian (ka)'
+      | 'Belarusian (be)'
+      | 'Tajik (tg)'
+      | 'Sindhi (sd)'
+      | 'Gujarati (gu)'
+      | 'Amharic (am)'
+      | 'Yiddish (yi)'
+      | 'Lao (lo)'
+      | 'Uzbek (uz)'
+      | 'Faroese (fo)'
+      | 'Haitian Creole (ht)'
+      | 'Pashto (ps)'
+      | 'Turkmen (tk)'
+      | 'Nynorsk (nn)'
+      | 'Maltese (mt)'
+      | 'Sanskrit (sa)'
+      | 'Luxembourgish (lb)'
+      | 'Myanmar (my)'
+      | 'Tibetan (bo)'
+      | 'Tagalog (tl)'
+      | 'Malagasy (mg)'
+      | 'Assamese (as)'
+      | 'Tatar (tt)'
+      | 'Hawaiian (haw)'
+      | 'Lingala (ln)'
+      | 'Hausa (ha)'
+      | 'Bashkir (ba)'
+      | 'Javanese (jw)'
+      | 'Sundanese (su)'
+      | DataLink
+    /* Initial prompt text to guide the model's style or continue a previous audio segment. The prompt should match the audio language. */
+    inferPrompt?: string | DataLink
+    /* Beam size to use for beam search (enables beam search if set) */
+    inferBeamSize?: number | DataLink
+    /* Translate the result to English */
+    inferTranslate?: boolean | DataLink
+    /* Max threads to use for inference */
+    inferMaxThreads?: number | DataLink
+    /* Output token-level timestamps in details outlet */
+    inferTokenTimestamps?: boolean | DataLink
+    /* Speaker diarization (Please use small.en-tdrz model) */
+    inferTdrz?: boolean | DataLink
+    /* Maximum segment length in characters */
+    inferMaxLength?: number | DataLink
+    /* Audio time offset in milliseconds */
+    inferOffset?: number | DataLink
+    /* Audio duration of audio to process in milliseconds */
+    inferDuration?: number | DataLink
+    /* The file URL or path to be inferred.  
+ It only supported `wav` format with 16kHz sample rate & single (mono) channel */
+    inferFileUrl?: string | DataLink
+    /* MD5 of file to be inferred */
+    inferFileMd5?: string | DataLink
+    /* Record duration in seconds. For performance, the value less than 30 seconds is recommended. */
+    inferRealtimeAudioSeconds?: number | DataLink
+    /* Optimize audio transcription performance by slicing audio samples when `Realtime Audio Seconds` > 30. */
+    inferRealtimeAudioSliceSeconds?: number | DataLink
+    /* Min duration of audio to start transcribe. Min: 0.5 seconds, Max: `Realtime Audio Slice Seconds`, Default: 1 second */
+    inferRealtimeAudioMinSeconds?: number | DataLink
+    /* Save recorded audio to file, the file path will be output to outlet. */
+    inferRealtimeSaveAudio?: boolean | DataLink
+    /* Start transcribe on recording when the audio volume is greater than the threshold by using VAD (Voice Activity Detection).  
+ The first VAD will be triggered after 2 second of recording. */
+    inferRealtimeVadEnabled?: boolean | DataLink
+    /* The length of the collected audio is used for VAD. (ms) */
+    inferRealtimeVadMs?: number | DataLink
+    /* VAD threshold */
+    inferRealtimeVadThold?: number | DataLink
+    /* Frequency to apply High-pass filter in VAD */
+    inferRealtimeVadFreqThold?: number | DataLink
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+    /* Event triggered when got transcribe result */
+    onTranscribed?: Array<EventAction>
+    /* Event triggered when transcribe realtime done */
+    onRealtimeStop?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Context details */
+    contextDetails?: () => Data
+    /* Is transcribing */
+    isTranscribing?: () => Data
+    /* Progress of transcribe audio (0-100) */
+    transcribeProgress?: () => Data
+    /* Inference result */
+    transcribeResult?: () => Data
+    /* Inference result details */
+    transcribeDetails?: () => Data
+    /* Recorded audio file path of transcribe realtime (if `Save Audio` is enabled) */
+    recordedPath?: () => Data
+  }
+}
+
+/* Local Speech-to-Text (STT) inference based on GGML and [whisper.cpp](https://github.com/ggerganov/whisper.cpp)  
+
+ ## Notice  
+ - iOS: Supported GPU acceleration, recommended use M1+ / A17+ chip device  
+ - macOS: Supported GPU acceleration, recommended use M1+ chip device  
+ - Android: Currently not supported GPU acceleration (Coming soon), recommended use Android 13+ system  
+ - Linux / Windows: Supported GPU acceleration, you can choose `vulkan` or `cuda` backend in Accel Variant property */
+export type GeneratorSpeechInference = Generator &
+  GeneratorSpeechInferenceDef & {
+    templateKey: 'GENERATOR_SPEECH_INFERENCE'
+    switches: Array<
+      SwitchDef &
+        GeneratorSpeechInferenceDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'contextDetails'
+                    | 'isTranscribing'
+                    | 'transcribeProgress'
+                    | 'transcribeResult'
+                    | 'transcribeDetails'
+                    | 'recordedPath'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/SpeechToTextOnnx.ts

@@ -0,0 +1,227 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Load the model */
+export type GeneratorOnnxSTTActionLoadModel = Action & {
+  __actionName: 'GENERATOR_ONNX_STT_LOAD_MODEL'
+}
+
+/* Inference */
+export type GeneratorOnnxSTTActionInfer = ActionWithParams & {
+  __actionName: 'GENERATOR_ONNX_STT_INFER'
+  params?: Array<{
+    input: 'audioUri'
+    value?: string | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Clean cache */
+export type GeneratorOnnxSTTActionCleanCache = Action & {
+  __actionName: 'GENERATOR_ONNX_STT_CLEAN_CACHE'
+}
+
+/* Release context */
+export type GeneratorOnnxSTTActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_ONNX_STT_RELEASE_CONTEXT'
+}
+
+interface GeneratorOnnxSTTDef {
+  /*
+Default property:
+{
+  "model": "whisper-base",
+  "modelType": "auto",
+  "returnTimestamps": "none",
+  "executionMode": "sequential"
+}
+  */
+  property?: {
+    /* Initialize the TTS context on generator initialization */
+    init?: boolean | DataLink
+    /* STT model */
+    model?: string | DataLink
+    /* Model type */
+    modelType?: string | DataLink
+    /* Quantize type */
+    quantizeType?:
+      | 'auto'
+      | 'none'
+      | 'fp16'
+      | 'q8'
+      | 'int8'
+      | 'uint8'
+      | 'q4'
+      | 'bnb4'
+      | 'q4f16'
+      | DataLink
+    /* Return timestamps */
+    returnTimestamps?: 'none' | 'enable' | 'word' | DataLink
+    /* Transcription language  
+  Not specifying the language will auto detect the language. */
+    language?:
+      | 'English'
+      | 'Chinese'
+      | 'German'
+      | 'Spanish'
+      | 'Russian'
+      | 'Korean'
+      | 'French'
+      | 'Japanese'
+      | 'Portuguese'
+      | 'Turkish'
+      | 'Polish'
+      | 'Catalan'
+      | 'Dutch'
+      | 'Arabic'
+      | 'Swedish'
+      | 'Italian'
+      | 'Indonesian'
+      | 'Hindi'
+      | 'Finnish'
+      | 'Vietnamese'
+      | 'Hebrew'
+      | 'Ukrainian'
+      | 'Greek'
+      | 'Malay'
+      | 'Czech'
+      | 'Romanian'
+      | 'Danish'
+      | 'Hungarian'
+      | 'Tamil'
+      | 'Norwegian'
+      | 'Thai'
+      | 'Urdu'
+      | 'Croatian'
+      | 'Bulgarian'
+      | 'Lithuanian'
+      | 'Latin'
+      | 'Maori'
+      | 'Malayalam'
+      | 'Welsh'
+      | 'Slovak'
+      | 'Telugu'
+      | 'Persian'
+      | 'Latvian'
+      | 'Bengali'
+      | 'Serbian'
+      | 'Azerbaijani'
+      | 'Slovenian'
+      | 'Kannada'
+      | 'Estonian'
+      | 'Macedonian'
+      | 'Breton'
+      | 'Basque'
+      | 'Icelandic'
+      | 'Armenian'
+      | 'Nepali'
+      | 'Mongolian'
+      | 'Bosnian'
+      | 'Kazakh'
+      | 'Albanian'
+      | 'Swahili'
+      | 'Galician'
+      | 'Marathi'
+      | 'Punjabi'
+      | 'Sinhala'
+      | 'Khmer'
+      | 'Shona'
+      | 'Yoruba'
+      | 'Somali'
+      | 'Afrikaans'
+      | 'Occitan'
+      | 'Georgian'
+      | 'Belarusian'
+      | 'Tajik'
+      | 'Sindhi'
+      | 'Gujarati'
+      | 'Amharic'
+      | 'Yiddish'
+      | 'Lao'
+      | 'Uzbek'
+      | 'Faroese'
+      | 'Haitian Creole'
+      | 'Pashto'
+      | 'Turkmen'
+      | 'Nynorsk'
+      | 'Maltese'
+      | 'Sanskrit'
+      | 'Luxembourgish'
+      | 'Myanmar'
+      | 'Tibetan'
+      | 'Tagalog'
+      | 'Malagasy'
+      | 'Assamese'
+      | 'Tatar'
+      | 'Hawaiian '
+      | 'Lingala'
+      | 'Hausa'
+      | 'Bashkir'
+      | 'Javanese'
+      | 'Sundanese'
+      | DataLink
+    /* Task */
+    task?: 'transcribe' | 'translate' | DataLink
+    /* Inferencing chunk length */
+    chunkLength?: number | DataLink
+    /* Executor candidates, descending order of priority  
+  Default will be xnnpack, wasm, cpu */
+    executors?:
+      | Array<'qnn' | 'dml' | 'nnapi' | 'xnnpack' | 'coreml' | 'cpu' | 'wasm' | 'webgpu' | DataLink>
+      | DataLink
+    /* Execution mode  
+  Usually when the model has many branches, setting this option to `parallel` will give you better performance. */
+    executionMode?: 'sequential' | 'parallel' | DataLink
+    /* QNN backend */
+    qnnBackend?: 'HTP' | 'HTA' | 'DSP' | 'GPU' | 'CPU' | DataLink
+    /* Enable FP16 for QNN HTP */
+    qnnHtpEnableFp16?: boolean | DataLink
+    /* Enable QNN debug */
+    qnnEnableDebug?: boolean | DataLink
+  }
+  events?: {
+    /* Event triggered when state change */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Inference result */
+    result?: () => Data
+    /* Inference result detail */
+    resultDetail?: () => Data
+  }
+}
+
+/* Local STT inference based on [transformers.js](https://huggingface.co/docs/transformers.js)  
+ You can use any converted model on HuggingFace. */
+export type GeneratorOnnxSTT = Generator &
+  GeneratorOnnxSTTDef & {
+    templateKey: 'GENERATOR_ONNX_STT'
+    switches: Array<
+      SwitchDef &
+        GeneratorOnnxSTTDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'contextState' | 'result' | 'resultDetail'
+                  value: any
+                }
+          }>
+        }
+    >
+  }