@fugood/bricks-project 2.22.0-beta.8 → 2.22.0

package/types/generators/RealtimeTranscription.ts

@@ -0,0 +1,180 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Start realtime transcription */
+export type GeneratorRealtimeTranscriptionActionStart = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_START'
+}
+
+/* Stop realtime transcription */
+export type GeneratorRealtimeTranscriptionActionStop = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_STOP'
+}
+
+/* Force move to next slice */
+export type GeneratorRealtimeTranscriptionActionNextSlice = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_NEXT_SLICE'
+}
+
+/* Reset transcriber state */
+export type GeneratorRealtimeTranscriptionActionReset = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_RESET'
+}
+
+interface GeneratorRealtimeTranscriptionDef {
+  /*
+Default property:
+{
+  "sttLivePolicy": "only-in-use",
+  "vadInferenceLivePolicy": "only-in-use",
+  "vadEnabled": true,
+  "audioSliceSec": 30,
+  "audioMinSec": 1,
+  "maxSlicesInMemory": 5,
+  "vadStrategy": "use-preset",
+  "vadPreset": "default",
+  "autoSliceOnSpeechEnd": true,
+  "autoSliceThreshold": 2,
+  "initialPrompt": "",
+  "promptPreviousSlices": false,
+  "saveAudio": true,
+  "testMode": false,
+  "testPlaybackSpeed": 1,
+  "testChunkDurationMs": 100,
+  "testLoop": false,
+  "beginTranscribeEnabled": false,
+  "beginTranscribeScript": "\/\* Global variable: inputs = { sliceIndex, duration, vadEvent, variables } \*\/\nreturn true"
+}
+  */
+  property?: {
+    /* STT Generator for Whisper context */
+    sttGeneratorId?: string | DataLink
+    /* STT Live Policy. If the policy is `only-in-use`, the STT context will be released when not in use. */
+    sttLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* VAD Inference Generator for voice activity detection */
+    vadInferenceGeneratorId?: string | DataLink
+    /* VAD Inference Live Policy. If the policy is `only-in-use`, the VAD Inference context will be released when not in use. */
+    vadInferenceLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* Enable VAD (Voice Activity Detection) */
+    vadEnabled?: boolean | DataLink
+    /* Audio slice duration in seconds */
+    audioSliceSec?: number | DataLink
+    /* Minimum audio duration to start transcription in seconds */
+    audioMinSec?: number | DataLink
+    /* Maximum number of slices to keep in memory */
+    maxSlicesInMemory?: number | DataLink
+    /* VAD Strategy */
+    vadStrategy?: 'use-preset' | 'use-generator-options' | DataLink
+    /* VAD preset configuration */
+    vadPreset?:
+      | 'default'
+      | 'sensitive'
+      | 'very-sensitive'
+      | 'conservative'
+      | 'very-conservative'
+      | 'continuous-speech'
+      | 'meeting'
+      | 'noisy-environment'
+      | DataLink
+    /* Auto slice on speech end */
+    autoSliceOnSpeechEnd?: boolean | DataLink
+    /* Auto slice threshold in seconds */
+    autoSliceThreshold?: number | DataLink
+    /* Initial prompt for transcription */
+    initialPrompt?: string | DataLink
+    /* Include previous slices in prompt */
+    promptPreviousSlices?: boolean | DataLink
+    /* Enable audio output saving (auto-generates file path) */
+    saveAudio?: boolean | DataLink
+    /* Use test mode with file simulation */
+    testMode?: boolean | DataLink
+    /* Test audio file path for simulation */
+    testFilePath?: string | DataLink
+    /* Test audio file hash */
+    testFileHash?: string | DataLink
+    /* Test audio file hash type */
+    testFileHashType?: string | DataLink
+    /* Test playback speed */
+    testPlaybackSpeed?: number | DataLink
+    /* Test chunk duration in milliseconds */
+    testChunkDurationMs?: number | DataLink
+    /* Loop test audio file */
+    testLoop?: boolean | DataLink
+    /* Enable Begin Transcribe Script to determine transcription */
+    beginTranscribeEnabled?: boolean | DataLink
+    /* Script to determine whether to proceed with transcription */
+    beginTranscribeScript?: string | DataLink
+    /* Data to be used in Begin Transcribe Script (object) */
+    beginTranscribeScriptData?: {} | DataLink
+  }
+  events?: {
+    /* Event triggered when transcription starts, processes, or ends */
+    onTranscribe?: Array<EventAction>
+    /* Event triggered on VAD (Voice Activity Detection) events */
+    onVad?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+    /* Event triggered when status changes */
+    onStatusChange?: Array<EventAction>
+    /* Event triggered when statistics update */
+    onStatsUpdate?: Array<EventAction>
+    /* Event triggered when transcription ends */
+    onEnd?: Array<EventAction>
+  }
+  outlets?: {
+    /* Is realtime transcription currently active */
+    isActive?: () => Data
+    /* Is currently transcribing audio */
+    isTranscribing?: () => Data
+    /* Current transcription results */
+    results?: () => Data
+    /* Current transcription result text */
+    resultText?: () => Data
+    /* Current statistics */
+    statistics?: () => Data
+    /* Latest transcribe event */
+    lastTranscribeEvent?: () => Data
+    /* Latest VAD event */
+    lastVadEvent?: () => Data
+    /* Audio output file path (auto-generated when saving audio) */
+    audioOutputPath?: () => Data
+  }
+}
+
+/* Realtime speech-to-text transcription using Whisper and VAD with live audio streaming */
+export type GeneratorRealtimeTranscription = Generator &
+  GeneratorRealtimeTranscriptionDef & {
+    templateKey: 'GENERATOR_REALTIME_TRANSCRIPTION'
+    switches: Array<
+      SwitchDef &
+        GeneratorRealtimeTranscriptionDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'isActive'
+                    | 'isTranscribing'
+                    | 'results'
+                    | 'resultText'
+                    | 'statistics'
+                    | 'lastTranscribeEvent'
+                    | 'lastVadEvent'
+                    | 'audioOutputPath'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/RerankerGgml.ts

@@ -0,0 +1,153 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Load the model */
+export type GeneratorRerankerActionLoadModel = Action & {
+  __actionName: 'GENERATOR_RERANKER_LOAD_MODEL'
+}
+
+/* Rerank documents based on query relevance */
+export type GeneratorRerankerActionRerank = ActionWithParams & {
+  __actionName: 'GENERATOR_RERANKER_RERANK'
+  params?: Array<
+    | {
+        input: 'query'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'documents'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Release context */
+export type GeneratorRerankerActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_RERANKER_RELEASE_CONTEXT'
+}
+
+interface GeneratorRerankerDef {
+  /*
+Default property:
+{
+  "init": false,
+  "contextSize": 512,
+  "batchSize": 512,
+  "uBatchSize": 512,
+  "accelVariant": "default",
+  "mainGpu": 0,
+  "gpuLayers": 0,
+  "useMlock": true,
+  "useMmap": true,
+  "normalize": 1
+}
+  */
+  property?: {
+    /* Initialize the Reranker context on generator initialization */
+    init?: boolean | DataLink
+    /* The URL or path of reranker model (GGUF format) */
+    modelUrl?: string | DataLink
+    /* Hash of reranker model */
+    modelHash?: string | DataLink
+    /* Hash type of reranker model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Context size (0 ~ 4096) (Default to 512) */
+    contextSize?: number | DataLink
+    /* Logical batch size for processing (default: 512) */
+    batchSize?: number | DataLink
+    /* Physical maximum batch size (default: 512) */
+    uBatchSize?: number | DataLink
+    /* Accelerator variant (default: default) */
+    accelVariant?:
+      | 'default'
+      | 'avx'
+      | 'avx2'
+      | 'avx512'
+      | 'metal'
+      | 'opencl'
+      | 'vulkan'
+      | 'cuda'
+      | 'rocm'
+      | DataLink
+    /* Main GPU index (default: 0) */
+    mainGpu?: number | DataLink
+    /* Number of layers to store in VRAM (default: 0) */
+    gpuLayers?: number | DataLink
+    /* Maximum number of threads to use (default: auto) */
+    maxThreads?: number | DataLink
+    /* Use mlock to keep model in memory (default: true) */
+    useMlock?: boolean | DataLink
+    /* Use mmap for model loading (default: true) */
+    useMmap?: boolean | DataLink
+    /* Query text for reranking */
+    query?: string | DataLink
+    /* Array of documents to rerank */
+    documents?: Array<string | DataLink> | DataLink
+    /* Normalize reranking scores (default: from model config) */
+    normalize?: number | DataLink | boolean | DataLink | DataLink
+    /* Maximum number of documents to return (default: unlimited) */
+    topK?: number | DataLink
+  }
+  events?: {
+    /* Event triggered when the reranker context state changes (loading, ready, error, released) */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when an error occurs during reranker operations */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* Current state of the reranker context (loading, ready, error, released) */
+    contextState?: () => Data
+    /* Loading progress of the reranker model (0-100) */
+    contextLoadProgress?: () => Data
+    /* Detailed information about the reranker context including instance ID and processing status */
+    contextDetails?: () => Data
+    /* Result of the reranking operation containing scored and ranked documents */
+    rerankResult?: () => Data
+    /* Boolean indicating whether the reranker is currently processing a request */
+    isProcessing?: () => Data
+  }
+}
+
+/* Local rerank based on GGML and [llama.cpp](https://github.com/ggerganov/llama.cpp)  
+
+ ## Notice  
+ - The device RAM must be larger than 8GB  
+ - iOS: Supported GPU acceleration, recommended use M1+ / A17+ chip device  
+ - macOS: Supported GPU acceleration, recommended use M1+ chip device  
+ - Android: Currently not supported GPU acceleration (Coming soon), recommended use Android 13+ system  
+ - Linux / Windows: Supported GPU acceleration, currently only Vulkan backend available */
+export type GeneratorReranker = Generator &
+  GeneratorRerankerDef & {
+    templateKey: 'GENERATOR_RERANKER'
+    switches: Array<
+      SwitchDef &
+        GeneratorRerankerDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'contextLoadProgress'
+                    | 'contextDetails'
+                    | 'rerankResult'
+                    | 'isProcessing'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/SerialPort.ts

@@ -0,0 +1,141 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Get available serial devices */
+export type GeneratorSerialPortActionGeneratorSerialGetDeviceList = Action & {
+  __actionName: 'GENERATOR_SERIAL_GET_DEVICE_LIST'
+}
+
+/* Attach device */
+export type GeneratorSerialPortActionGeneratorSerialAttach = Action & {
+  __actionName: 'GENERATOR_SERIAL_ATTACH'
+}
+
+/* Dettach device */
+export type GeneratorSerialPortActionGeneratorSerialDetach = Action & {
+  __actionName: 'GENERATOR_SERIAL_DETACH'
+}
+
+/* Send string data (ASCII) */
+export type GeneratorSerialPortActionGeneratorSerialSendString = ActionWithParams & {
+  __actionName: 'GENERATOR_SERIAL_SEND_STRING'
+  params?: Array<
+    | {
+        input: 'generatorSerialStringValue'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'generatorSerialStringEncoding'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Send hex string (e.g. 0123456789ABCDEF) */
+export type GeneratorSerialPortActionGeneratorSerialSendHex = ActionWithParams & {
+  __actionName: 'GENERATOR_SERIAL_SEND_HEX'
+  params?: Array<{
+    input: 'generatorSerialHexValue'
+    value?: string | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+interface GeneratorSerialPortDef {
+  /*
+Default property:
+{
+  "attachOnInit": false,
+  "driver": "fd",
+  "parityBit": "none",
+  "dataBits": 8,
+  "stopBits": 1,
+  "commMode": "raw"
+}
+  */
+  property?: {
+    /* Try attach on generator initialized  On web require user activation to attach device */
+    attachOnInit?: boolean | DataLink
+    /* The serial device driver */
+    driver?: 'fd' | 'usb' | DataLink
+    /* The serial device path  e.g. /dev/ttyS0 or /dev/bus/usb/001/001  For desktop and web is device index number */
+    path?: string | DataLink
+    /* The serial USB vendor id (autoconnect first) */
+    vendorId?: number | DataLink
+    /* The serial USB product id */
+    productId?: number | DataLink
+    /* The super user binary path (For Android root permission) */
+    suPath?: string | DataLink
+    /* The serial device baud rate */
+    baudRate?: number | DataLink
+    /* The serial device parity bit */
+    parityBit?: 'none' | 'odd' | 'even' | DataLink
+    /* The serial device data bits */
+    dataBits?: number | DataLink
+    /* The serial device stop bits */
+    stopBits?: number | DataLink
+    /* The serial device communication mode */
+    commMode?: 'raw' | 'EDC' | 'Regex Match' | DataLink
+    /* The regex match pattern for packets */
+    regexMatch?: string | DataLink
+  }
+  events?: {
+    /* Event for attach the device */
+    generatorSerialOnAttach?: Array<EventAction>
+    /* Event for detach the device */
+    generatorSerialOnDetach?: Array<EventAction>
+    /* Event on error */
+    generatorSerialOnError?: Array<EventAction>
+    /* Event for receive data */
+    generatorSerialOnData?: Array<EventAction>
+  }
+  outlets?: {
+    /* Available serial devices */
+    generatorSerialDeviceList?: () => Data
+    /* Is serial open */
+    generatorSerialIsOpen?: () => Data
+    /* Last rerceive data (Base64) */
+    generatorSerialLastDataBase64?: () => Data
+    /* Last rerceive plain text */
+    generatorSerialLastDataString?: () => Data
+    /* Error message */
+    generatorSerialErrorMessage?: () => Data
+  }
+}
+
+/* Serial Port Generator (Unsupported iOS) */
+export type GeneratorSerialPort = Generator &
+  GeneratorSerialPortDef & {
+    templateKey: 'GENERATOR_SERIAL_PORT'
+    switches: Array<
+      SwitchDef &
+        GeneratorSerialPortDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'generatorSerialDeviceList'
+                    | 'generatorSerialIsOpen'
+                    | 'generatorSerialLastDataBase64'
+                    | 'generatorSerialLastDataString'
+                    | 'generatorSerialErrorMessage'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/SoundPlayer.ts

@@ -0,0 +1,86 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Start playing sound */
+export type GeneratorSoundPlayerActionPlay = Action & {
+  __actionName: 'GENERATOR_SOUND_PLAYER_PLAY'
+}
+
+/* Pause playing sound */
+export type GeneratorSoundPlayerActionPause = Action & {
+  __actionName: 'GENERATOR_SOUND_PLAYER_PAUSE'
+}
+
+/* Resume playing sound from pause */
+export type GeneratorSoundPlayerActionResume = Action & {
+  __actionName: 'GENERATOR_SOUND_PLAYER_RESUME'
+}
+
+/* Stop playing sound */
+export type GeneratorSoundPlayerActionRelease = Action & {
+  __actionName: 'GENERATOR_SOUND_PLAYER_RELEASE'
+}
+
+interface GeneratorSoundPlayerDef {
+  /*
+Default property:
+{
+  "loop": false,
+  "volume": 100
+}
+  */
+  property?: {
+    /* Sound file path */
+    filePath?: string | DataLink
+    /* MD5 */
+    md5?: string | DataLink
+    /* Repeat playback */
+    loop?: boolean | DataLink
+    /* Sound volume (0 - 100) */
+    volume?: number | DataLink
+  }
+  events?: {
+    /* Sound file loaded successfully */
+    onLoad?: Array<EventAction>
+    /* Sound file load error */
+    onLoadError?: Array<EventAction>
+    /* Sound playback complete */
+    onPlay?: Array<EventAction>
+    /* Sound file playback end */
+    onEnd?: Array<EventAction>
+  }
+  outlets?: {
+    /* Whether the sound is playing */
+    isPlaying?: () => Data
+  }
+}
+
+/* Play sound, see supported formats at https://developer.android.com/guide/topics/media/media-formats */
+export type GeneratorSoundPlayer = Generator &
+  GeneratorSoundPlayerDef & {
+    templateKey: 'GENERATOR_SOUND_PLAYER'
+    switches: Array<
+      SwitchDef &
+        GeneratorSoundPlayerDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'isPlaying'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/SoundRecorder.ts

@@ -0,0 +1,113 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Start record */
+export type GeneratorSoundRecorderActionStart = Action & {
+  __actionName: 'GENERATOR_SOUND_RECORDER_START'
+}
+
+/* Pause record */
+export type GeneratorSoundRecorderActionPause = Action & {
+  __actionName: 'GENERATOR_SOUND_RECORDER_PAUSE'
+}
+
+/* Resume record */
+export type GeneratorSoundRecorderActionResume = Action & {
+  __actionName: 'GENERATOR_SOUND_RECORDER_RESUME'
+}
+
+/* Stop record */
+export type GeneratorSoundRecorderActionStop = Action & {
+  __actionName: 'GENERATOR_SOUND_RECORDER_STOP'
+}
+
+interface GeneratorSoundRecorderDef {
+  /*
+Default property:
+{
+  "init": false,
+  "sampleRate": 32000,
+  "bitDepth": "16bit",
+  "channels": "stereo",
+  "fileRetainTime": 10000,
+  "autoCuttingEnable": false,
+  "autoCuttingTimeLimit": 0,
+  "autoCuttingVocalFreqMin": 75,
+  "autoCuttingVocalFreqMax": 900,
+  "autoCuttingFreqClarity": 0.5,
+  "autoCuttingSpeakSpacing": 0.5
+}
+  */
+  property?: {
+    /* Start record on init (Need user trigger on Web) */
+    init?: boolean | DataLink
+    /* Record sample rate */
+    sampleRate?: number | DataLink
+    /* PCM bit depth */
+    bitDepth?: '8bit' | '16bit' | DataLink
+    /* Audio channels */
+    channels?: 'mono' | 'stereo' | DataLink
+    /* Record file retain time, avoid memory exhausted */
+    fileRetainTime?: number | DataLink
+    /* Enable record auto cutting */
+    autoCuttingEnable?: boolean | DataLink
+    /* Fragment time limit */
+    autoCuttingTimeLimit?: number | DataLink
+    /* Minimal vocal frequency of cutting fragments */
+    autoCuttingVocalFreqMin?: number | DataLink
+    /* Maximal vocal frequency of cutting fragments */
+    autoCuttingVocalFreqMax?: number | DataLink
+    /* Minimum frequency clarity threshold on speaking (0 ~ 1) */
+    autoCuttingFreqClarity?: number | DataLink
+    /* Minimum volume threshold on speaking */
+    autoCuttingVolumeThreshold?: number | DataLink
+    /* Speak spacing of cutting fragments */
+    autoCuttingSpeakSpacing?: number | DataLink
+  }
+  events?: {
+    /* Event for chunk data (Base64 encoded PCM) */
+    chunk?: Array<EventAction>
+    /* Event for recorded file */
+    recorded?: Array<EventAction>
+  }
+  outlets?: {
+    /* Is recording */
+    isRecording?: () => Data
+    /* Recorded file path */
+    recordedPath?: () => Data
+    /* Current auto-cutting detected volume */
+    volume?: () => Data
+    /* Error message */
+    errorMessage?: () => Data
+  }
+}
+
+/* Record audio (Microphone) */
+export type GeneratorSoundRecorder = Generator &
+  GeneratorSoundRecorderDef & {
+    templateKey: 'GENERATOR_SOUND_RECORDER'
+    switches: Array<
+      SwitchDef &
+        GeneratorSoundRecorderDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'isRecording' | 'recordedPath' | 'volume' | 'errorMessage'
+                  value: any
+                }
+          }>
+        }
+    >
+  }