@fugood/bricks-ctor 2.24.0-beta.40

package/types/generators/SoundPlayer.ts

@@ -0,0 +1,94 @@
+/* Auto generated by build script
+ *
+ * Play sound, see supported formats at https://developer.android.com/guide/topics/media/media-formats
+ */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+import type { TemplateEventPropsMap } from '../../utils/event-props'
+
+/* Start playing sound */
+export type GeneratorSoundPlayerActionPlay = Action & {
+  __actionName: 'GENERATOR_SOUND_PLAYER_PLAY'
+}
+
+/* Pause playing sound */
+export type GeneratorSoundPlayerActionPause = Action & {
+  __actionName: 'GENERATOR_SOUND_PLAYER_PAUSE'
+}
+
+/* Resume playing sound from pause */
+export type GeneratorSoundPlayerActionResume = Action & {
+  __actionName: 'GENERATOR_SOUND_PLAYER_RESUME'
+}
+
+/* Stop playing sound */
+export type GeneratorSoundPlayerActionRelease = Action & {
+  __actionName: 'GENERATOR_SOUND_PLAYER_RELEASE'
+}
+
+interface GeneratorSoundPlayerDef {
+  /*
+Default property:
+{
+  "loop": false,
+  "volume": 100
+}
+  */
+  property?: {
+    /* Sound file path */
+    filePath?: string | DataLink
+    /* MD5 */
+    md5?: string | DataLink
+    /* Repeat playback */
+    loop?: boolean | DataLink
+    /* Sound volume (0 - 100) */
+    volume?: number | DataLink
+  }
+  events?: {
+    /* Sound file loaded successfully */
+    onLoad?: Array<EventAction>
+    /* Sound file load error */
+    onLoadError?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['SoundPlayer']['onLoadError']>
+    >
+    /* Sound playback complete */
+    onPlay?: Array<EventAction>
+    /* Sound file playback end */
+    onEnd?: Array<EventAction>
+  }
+  outlets?: {
+    /* Whether the sound is playing */
+    isPlaying?: () => Data<any>
+  }
+}
+
+/* Play sound, see supported formats at https://developer.android.com/guide/topics/media/media-formats */
+export type GeneratorSoundPlayer = Generator &
+  GeneratorSoundPlayerDef & {
+    templateKey: 'GENERATOR_SOUND_PLAYER'
+    switches?: Array<
+      SwitchDef &
+        GeneratorSoundPlayerDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'isPlaying'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/SoundRecorder.ts

@@ -0,0 +1,130 @@
+/* Auto generated by build script
+ *
+ * Record audio (Microphone)
+ */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+import type { TemplateEventPropsMap } from '../../utils/event-props'
+
+/* Start record */
+export type GeneratorSoundRecorderActionStart = Action & {
+  __actionName: 'GENERATOR_SOUND_RECORDER_START'
+}
+
+/* Pause record */
+export type GeneratorSoundRecorderActionPause = Action & {
+  __actionName: 'GENERATOR_SOUND_RECORDER_PAUSE'
+}
+
+/* Resume record */
+export type GeneratorSoundRecorderActionResume = Action & {
+  __actionName: 'GENERATOR_SOUND_RECORDER_RESUME'
+}
+
+/* Stop record */
+export type GeneratorSoundRecorderActionStop = Action & {
+  __actionName: 'GENERATOR_SOUND_RECORDER_STOP'
+}
+
+/* List available microphone devices (Web / Desktop only) */
+export type GeneratorSoundRecorderActionListDevices = Action & {
+  __actionName: 'GENERATOR_SOUND_RECORDER_LIST_DEVICES'
+}
+
+interface GeneratorSoundRecorderDef {
+  /*
+Default property:
+{
+  "init": false,
+  "sampleRate": 32000,
+  "bitDepth": "16bit",
+  "channels": "stereo",
+  "fileRetainTime": 10000,
+  "autoCuttingEnable": false,
+  "autoCuttingTimeLimit": 0,
+  "autoCuttingVocalFreqMin": 75,
+  "autoCuttingVocalFreqMax": 900,
+  "autoCuttingFreqClarity": 0.5,
+  "autoCuttingSpeakSpacing": 0.5
+}
+  */
+  property?: {
+    /* Start record on init (Need user trigger on Web) */
+    init?: boolean | DataLink
+    /* Specifies the device ID to use for audio input.
+  Web / Desktop: device ID string from enumerateDevices  
+  Android: audio source name - DEFAULT, MIC, VOICE_UPLINK, VOICE_DOWNLINK, VOICE_CALL, CAMCORDER, VOICE_RECOGNITION, VOICE_COMMUNICATION, UNPROCESSED */
+    deviceId?: string | DataLink
+    /* Record sample rate */
+    sampleRate?: number | DataLink
+    /* PCM bit depth */
+    bitDepth?: '8bit' | '16bit' | DataLink
+    /* Audio channels */
+    channels?: 'mono' | 'stereo' | DataLink
+    /* Record file retain time, avoid memory exhausted */
+    fileRetainTime?: number | DataLink
+    /* Enable record auto cutting */
+    autoCuttingEnable?: boolean | DataLink
+    /* Fragment time limit */
+    autoCuttingTimeLimit?: number | DataLink
+    /* Minimal vocal frequency of cutting fragments */
+    autoCuttingVocalFreqMin?: number | DataLink
+    /* Maximal vocal frequency of cutting fragments */
+    autoCuttingVocalFreqMax?: number | DataLink
+    /* Minimum frequency clarity threshold on speaking (0 ~ 1) */
+    autoCuttingFreqClarity?: number | DataLink
+    /* Minimum volume threshold on speaking */
+    autoCuttingVolumeThreshold?: number | DataLink
+    /* Speak spacing of cutting fragments */
+    autoCuttingSpeakSpacing?: number | DataLink
+  }
+  events?: {
+    /* Event for chunk data (Base64 encoded PCM) */
+    chunk?: Array<EventAction<string & keyof TemplateEventPropsMap['SoundRecorder']['chunk']>>
+    /* Event for recorded file */
+    recorded?: Array<EventAction<string & keyof TemplateEventPropsMap['SoundRecorder']['recorded']>>
+  }
+  outlets?: {
+    /* Is recording */
+    isRecording?: () => Data<boolean>
+    /* Recorded file path */
+    recordedPath?: () => Data<string>
+    /* Current auto-cutting detected volume */
+    volume?: () => Data<number>
+    /* Error message */
+    errorMessage?: () => Data<string>
+    /* Available microphone devices (Web / Desktop only) */
+    devices?: () => Data<Array<any>>
+  }
+}
+
+/* Record audio (Microphone) */
+export type GeneratorSoundRecorder = Generator &
+  GeneratorSoundRecorderDef & {
+    templateKey: 'GENERATOR_SOUND_RECORDER'
+    switches?: Array<
+      SwitchDef &
+        GeneratorSoundRecorderDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'isRecording' | 'recordedPath' | 'volume' | 'errorMessage' | 'devices'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/SpeechToTextGgml.ts

@@ -0,0 +1,415 @@
+/* Auto generated by build script
+ *
+ * Local Speech-to-Text (STT) inference based on GGML and [whisper.cpp](https://github.com/ggerganov/whisper.cpp)
+ *
+ *  ## Notice
+ *  - iOS: Supported GPU acceleration, recommended use M1+ / A17+ chip device
+ *  - macOS: Supported GPU acceleration, recommended use M1+ chip device
+ *  - Android: Currently not supported GPU acceleration (Coming soon), recommended use Android 13+ system
+ *  - Linux / Windows: Supported GPU acceleration, you can choose `vulkan` or `cuda` backend in Accel Variant property
+ */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+import type { TemplateEventPropsMap } from '../../utils/event-props'
+
+/* Load the model */
+export type GeneratorSpeechInferenceActionLoadModel = Action & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_LOAD_MODEL'
+}
+
+/* Transcribe audio file. You can provide `File URL` property, if not provided, it will use the default `File URL` */
+export type GeneratorSpeechInferenceActionTranscribeFile = ActionWithParams & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_FILE'
+  params?: Array<
+    | {
+        input: 'fileUrl'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'beamSize'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'language'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'translate'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Transcribe audio data. Currently only support base64 encoded audio data (16-bit PCM, mono, 16kHz) */
+export type GeneratorSpeechInferenceActionTranscribeData = ActionWithParams & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_DATA'
+  params?: Array<
+    | {
+        input: 'data'
+        value?: any | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'beamSize'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'language'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'translate'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Stop current transcription */
+export type GeneratorSpeechInferenceActionTranscribeStop = Action & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_STOP'
+}
+
+/* Clear downloaded files (model, audio) & current jobs */
+export type GeneratorSpeechInferenceActionClearDownload = Action & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_CLEAR_DOWNLOAD'
+}
+
+/* Release context */
+export type GeneratorSpeechInferenceActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_RELEASE_CONTEXT'
+}
+
+interface GeneratorSpeechInferenceDef {
+  /*
+Default property:
+{
+  "init": false,
+  "accelVariant": "default",
+  "modelName": "base-q8_0",
+  "modelUseCoreML": false,
+  "modelUseGPU": true,
+  "modelUseFlashAttn": false,
+  "inferLanguage": "Auto"
+}
+  */
+  property?: {
+    /* Initialize the Whisper context on generator initialization
+ Please note that it will take some RAM depending on the model size */
+    init?: boolean | DataLink
+    /* Accelerator variant (Only for desktop)
+ `default` - CPU / Metal (macOS)  
+ `vulkan` - Use Vulkan  
+ `cuda` - Use CUDA */
+    accelVariant?: 'default' | 'vulkan' | 'cuda' | DataLink
+    /* Use model name,  the model download progress will be done in preload stage or  the generator initialization stage.
+ We used `ggml` format model, please refer to https://huggingface.co/BricksDisplay/whisper-ggml  
+ You can also choose `custom` option and set `Model URL` and `Model MD5` to use your own model */
+    modelName?:
+      | 'custom'
+      | 'tiny'
+      | 'tiny-q5_1'
+      | 'tiny-q8_0'
+      | 'tiny.en'
+      | 'tiny.en-q5_1'
+      | 'tiny.en-q8_0'
+      | 'base'
+      | 'base-q5_1'
+      | 'base-q8_0'
+      | 'base.en'
+      | 'base.en-q5_1'
+      | 'base.en-q8_0'
+      | 'small'
+      | 'small-q5_1'
+      | 'small-q8_0'
+      | 'small.en'
+      | 'small.en-q5_1'
+      | 'small.en-q8_0'
+      | 'medium'
+      | 'medium-q5_0'
+      | 'medium-q8_0'
+      | 'medium.en'
+      | 'medium.en-q5_0'
+      | 'medium.en-q8_0'
+      | 'large-v1'
+      | 'large-v2'
+      | 'large-v2-q5_0'
+      | 'large-v2-q8_0'
+      | 'large-v3'
+      | 'large-v3-q5_0'
+      | 'large-v3-turbo'
+      | 'large-v3-turbo-q5_0'
+      | 'large-v3-turbo-q8_0'
+      | 'small.en-tdrz'
+      | 'small.en-tdrz-q5_1'
+      | 'small.en-tdrz-q8_0'
+      | 'distil-small.en'
+      | 'distil-small.en-q5_1'
+      | 'distil-small.en-q8_0'
+      | 'distil-medium.en'
+      | 'distil-medium.en-q5_0'
+      | 'distil-medium.en-q8_0'
+      | 'distil-large-v3'
+      | 'distil-large-v3-q5_0'
+      | 'distil-large-v3-q8_0'
+      | DataLink
+    /* The URL or path of model
+ We used `ggml` format model, please refer to https://github.com/ggerganov/whisper.cpp/tree/master/models */
+    modelUrl?: string | DataLink
+    /* Hash type of model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of model */
+    modelHash?: string | DataLink
+    /* [Unstable] iOS: Use CoreML model for inference */
+    modelUseCoreML?: boolean | DataLink
+    /* Use GPU Acceleration for inference. Currently iOS only, if it's enabled, Core ML option will be ignored. */
+    modelUseGPU?: boolean | DataLink
+    /* Use Flash Attention for inference (Recommended with GPU enabled) */
+    modelUseFlashAttn?: boolean | DataLink
+    /* The language of the file to be inferred */
+    inferLanguage?:
+      | 'Auto'
+      | 'English (en)'
+      | 'Chinese (zh)'
+      | 'German (de)'
+      | 'Spanish (es)'
+      | 'Russian (ru)'
+      | 'Korean (ko)'
+      | 'French (fr)'
+      | 'Japanese (ja)'
+      | 'Portuguese (pt)'
+      | 'Turkish (tr)'
+      | 'Polish (pl)'
+      | 'Catalan (ca)'
+      | 'Dutch (nl)'
+      | 'Arabic (ar)'
+      | 'Swedish (sv)'
+      | 'Italian (it)'
+      | 'Indonesian (id)'
+      | 'Hindi (hi)'
+      | 'Finnish (fi)'
+      | 'Vietnamese (vi)'
+      | 'Hebrew (he)'
+      | 'Ukrainian (uk)'
+      | 'Greek (el)'
+      | 'Malay (ms)'
+      | 'Czech (cs)'
+      | 'Romanian (ro)'
+      | 'Danish (da)'
+      | 'Hungarian (hu)'
+      | 'Tamil (ta)'
+      | 'Norwegian (no)'
+      | 'Thai (th)'
+      | 'Urdu (ur)'
+      | 'Croatian (hr)'
+      | 'Bulgarian (bg)'
+      | 'Lithuanian (lt)'
+      | 'Latin (la)'
+      | 'Maori (mi)'
+      | 'Malayalam (ml)'
+      | 'Welsh (cy)'
+      | 'Slovak (sk)'
+      | 'Telugu (te)'
+      | 'Persian (fa)'
+      | 'Latvian (lv)'
+      | 'Bengali (bn)'
+      | 'Serbian (sr)'
+      | 'Azerbaijani (az)'
+      | 'Slovenian (sl)'
+      | 'Kannada (kn)'
+      | 'Estonian (et)'
+      | 'Macedonian (mk)'
+      | 'Breton (br)'
+      | 'Basque (eu)'
+      | 'Icelandic (is)'
+      | 'Armenian (hy)'
+      | 'Nepali (ne)'
+      | 'Mongolian (mn)'
+      | 'Bosnian (bs)'
+      | 'Kazakh (kk)'
+      | 'Albanian (sq)'
+      | 'Swahili (sw)'
+      | 'Galician (gl)'
+      | 'Marathi (mr)'
+      | 'Punjabi (pa)'
+      | 'Sinhala (si)'
+      | 'Khmer (km)'
+      | 'Shona (sn)'
+      | 'Yoruba (yo)'
+      | 'Somali (so)'
+      | 'Afrikaans (af)'
+      | 'Occitan (oc)'
+      | 'Georgian (ka)'
+      | 'Belarusian (be)'
+      | 'Tajik (tg)'
+      | 'Sindhi (sd)'
+      | 'Gujarati (gu)'
+      | 'Amharic (am)'
+      | 'Yiddish (yi)'
+      | 'Lao (lo)'
+      | 'Uzbek (uz)'
+      | 'Faroese (fo)'
+      | 'Haitian Creole (ht)'
+      | 'Pashto (ps)'
+      | 'Turkmen (tk)'
+      | 'Nynorsk (nn)'
+      | 'Maltese (mt)'
+      | 'Sanskrit (sa)'
+      | 'Luxembourgish (lb)'
+      | 'Myanmar (my)'
+      | 'Tibetan (bo)'
+      | 'Tagalog (tl)'
+      | 'Malagasy (mg)'
+      | 'Assamese (as)'
+      | 'Tatar (tt)'
+      | 'Hawaiian (haw)'
+      | 'Lingala (ln)'
+      | 'Hausa (ha)'
+      | 'Bashkir (ba)'
+      | 'Javanese (jw)'
+      | 'Sundanese (su)'
+      | DataLink
+    /* Initial prompt text to guide the model's style or continue a previous audio segment. The prompt should match the audio language. */
+    inferPrompt?: string | DataLink
+    /* Beam size to use for beam search (enables beam search if set) */
+    inferBeamSize?: number | DataLink
+    /* Translate the result to English */
+    inferTranslate?: boolean | DataLink
+    /* Max threads to use for inference */
+    inferMaxThreads?: number | DataLink
+    /* Output token-level timestamps in details outlet */
+    inferTokenTimestamps?: boolean | DataLink
+    /* Speaker diarization (Please use small.en-tdrz model) */
+    inferTdrz?: boolean | DataLink
+    /* Maximum segment length in characters */
+    inferMaxLength?: number | DataLink
+    /* Audio time offset in milliseconds */
+    inferOffset?: number | DataLink
+    /* Audio duration of audio to process in milliseconds */
+    inferDuration?: number | DataLink
+    /* The file URL or path to be inferred.
+ It only supported `wav` format with 16kHz sample rate & single (mono) channel */
+    inferFileUrl?: string | DataLink
+    /* MD5 of file to be inferred */
+    inferFileMd5?: string | DataLink
+    /* Buttress connection settings for remote inference */
+    buttressConnectionSettings?:
+      | DataLink
+      | {
+          enabled?: boolean | DataLink
+          url?: string | DataLink
+          fallbackType?: 'use-local' | 'no-op' | DataLink
+          strategy?: 'prefer-local' | 'prefer-buttress' | 'prefer-best' | DataLink
+        }
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['SpeechInference']['onContextStateChange']>
+    >
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction<string & keyof TemplateEventPropsMap['SpeechInference']['onError']>>
+    /* Event triggered when got transcribe result */
+    onTranscribed?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['SpeechInference']['onTranscribed']>
+    >
+    /* Event triggered when transcribe realtime done */
+    onRealtimeStop?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data<string>
+    /* Context details */
+    contextDetails?: () => Data<{
+      state?: string
+      contextId?: string
+      gpu?: string
+      reasonNoGPU?: string
+      [key: string]: any
+    }>
+    /* Is transcribing */
+    isTranscribing?: () => Data<boolean>
+    /* Progress of transcribe audio (0-100) */
+    transcribeProgress?: () => Data<number>
+    /* Inference result */
+    transcribeResult?: () => Data<string>
+    /* Inference result details */
+    transcribeDetails?: () => Data<{
+      result?: string
+      segments?: Array<{
+        start?: number
+        end?: number
+        confidence?: number
+        t0?: number
+        t1?: number
+        [key: string]: any
+      }>
+      time?: number
+      startTime?: number
+      endTime?: number
+      [key: string]: any
+    }>
+    /* Recorded audio file path of transcribe realtime (if `Save Audio` is enabled) */
+    recordedPath?: () => Data<string>
+  }
+}
+
+/* Local Speech-to-Text (STT) inference based on GGML and [whisper.cpp](https://github.com/ggerganov/whisper.cpp)
+
+ ## Notice  
+ - iOS: Supported GPU acceleration, recommended use M1+ / A17+ chip device  
+ - macOS: Supported GPU acceleration, recommended use M1+ chip device  
+ - Android: Currently not supported GPU acceleration (Coming soon), recommended use Android 13+ system  
+ - Linux / Windows: Supported GPU acceleration, you can choose `vulkan` or `cuda` backend in Accel Variant property */
+export type GeneratorSpeechInference = Generator &
+  GeneratorSpeechInferenceDef & {
+    templateKey: 'GENERATOR_SPEECH_INFERENCE'
+    switches?: Array<
+      SwitchDef &
+        GeneratorSpeechInferenceDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'contextDetails'
+                    | 'isTranscribing'
+                    | 'transcribeProgress'
+                    | 'transcribeResult'
+                    | 'transcribeDetails'
+                    | 'recordedPath'
+                  value: any
+                }
+          }>
+        }
+    >
+  }