@fugood/bricks-ctor 2.24.0-beta.40

package/types/generators/RealtimeTranscription.ts

@@ -0,0 +1,279 @@
+/* Auto generated by build script
+ *
+ * Realtime speech-to-text transcription using Whisper and VAD with live audio streaming
+ */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+import type { TemplateEventPropsMap } from '../../utils/event-props'
+
+/* Start realtime transcription */
+export type GeneratorRealtimeTranscriptionActionStart = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_START'
+}
+
+/* Stop realtime transcription */
+export type GeneratorRealtimeTranscriptionActionStop = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_STOP'
+}
+
+/* Force move to next slice */
+export type GeneratorRealtimeTranscriptionActionNextSlice = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_NEXT_SLICE'
+}
+
+/* Reset transcriber state */
+export type GeneratorRealtimeTranscriptionActionReset = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_RESET'
+}
+
+/* List available microphone devices (Web / Desktop only) */
+export type GeneratorRealtimeTranscriptionActionListDevices = Action & {
+  __actionName: 'GENERATOR_REALTIME_TRANSCRIPTION_LIST_DEVICES'
+}
+
+interface GeneratorRealtimeTranscriptionDef {
+  /*
+Default property:
+{
+  "sttLivePolicy": "only-in-use",
+  "vadInferenceLivePolicy": "only-in-use",
+  "vadEnabled": true,
+  "audioSliceSec": 30,
+  "audioMinSec": 1,
+  "maxSlicesInMemory": 5,
+  "transcribeProcessingPauseMs": 500,
+  "initTranscribeAfterMs": 500,
+  "vadStrategy": "use-preset",
+  "vadPreset": "default",
+  "initialPrompt": "",
+  "promptPreviousSlices": false,
+  "saveAudio": true,
+  "testMode": false,
+  "testPlaybackSpeed": 1,
+  "testChunkDurationMs": 100,
+  "testLoop": false,
+  "beginTranscribeEnabled": false,
+  "beginTranscribeScript": "\/\* Global variable: inputs = { sliceIndex, duration, vadEvent, variables } \*\/\nreturn true"
+}
+  */
+  property?: {
+    /* Specifies the device ID to use for audio input.
+  Web / Desktop: device ID string from enumerateDevices  
+  Android: audio source name - DEFAULT, MIC, VOICE_UPLINK, VOICE_DOWNLINK, VOICE_CALL, CAMCORDER, VOICE_RECOGNITION, VOICE_COMMUNICATION, UNPROCESSED */
+    deviceId?: string | DataLink
+    /* STT Generator for Whisper context */
+    sttGeneratorId?: string | DataLink | (() => Generator)
+    /* STT Live Policy. If the policy is `only-in-use`, the STT context will be released when not in use. */
+    sttLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* VAD Inference Generator for voice activity detection */
+    vadInferenceGeneratorId?: string | DataLink | (() => Generator)
+    /* VAD Inference Live Policy. If the policy is `only-in-use`, the VAD Inference context will be released when not in use. */
+    vadInferenceLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* Enable VAD (Voice Activity Detection) */
+    vadEnabled?: boolean | DataLink
+    /* Audio slice duration in seconds */
+    audioSliceSec?: number | DataLink
+    /* Minimum audio duration to start transcription in seconds */
+    audioMinSec?: number | DataLink
+    /* Maximum number of slices to keep in memory */
+    maxSlicesInMemory?: number | DataLink
+    /* Transcribe processing interval in milliseconds */
+    transcribeProcessingPauseMs?: number | DataLink
+    /* Transcribe processing init after pause in milliseconds */
+    initTranscribeAfterMs?: number | DataLink
+    /* VAD Strategy */
+    vadStrategy?: 'use-preset' | 'use-generator-options' | DataLink
+    /* VAD preset configuration */
+    vadPreset?:
+      | 'default'
+      | 'sensitive'
+      | 'very-sensitive'
+      | 'conservative'
+      | 'very-conservative'
+      | 'continuous-speech'
+      | 'meeting'
+      | 'noisy-environment'
+      | DataLink
+    /* Initial prompt for transcription */
+    initialPrompt?: string | DataLink
+    /* Include previous slices in prompt */
+    promptPreviousSlices?: boolean | DataLink
+    /* Enable audio output saving (auto-generates file path) */
+    saveAudio?: boolean | DataLink
+    /* Use test mode with file simulation */
+    testMode?: boolean | DataLink
+    /* Test audio file path for simulation */
+    testFilePath?: string | DataLink
+    /* Test audio file hash */
+    testFileHash?: string | DataLink
+    /* Test audio file hash type */
+    testFileHashType?: string | DataLink
+    /* Test playback speed */
+    testPlaybackSpeed?: number | DataLink
+    /* Test chunk duration in milliseconds */
+    testChunkDurationMs?: number | DataLink
+    /* Loop test audio file */
+    testLoop?: boolean | DataLink
+    /* Enable Begin Transcribe Script to determine transcription */
+    beginTranscribeEnabled?: boolean | DataLink
+    /* Script to determine whether to proceed with transcription */
+    beginTranscribeScript?: string | DataLink
+    /* Data to be used in Begin Transcribe Script (object) */
+    beginTranscribeScriptData?: {} | DataLink
+  }
+  events?: {
+    /* Event triggered when transcription starts, processes, or ends */
+    onTranscribe?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['RealtimeTranscription']['onTranscribe']>
+    >
+    /* Event triggered on VAD (Voice Activity Detection) events */
+    onVad?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['RealtimeTranscription']['onVad']>
+    >
+    /* Event triggered when error occurs */
+    onError?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['RealtimeTranscription']['onError']>
+    >
+    /* Event triggered when status changes */
+    onStatusChange?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['RealtimeTranscription']['onStatusChange']>
+    >
+    /* Event triggered when statistics update */
+    onStatsUpdate?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['RealtimeTranscription']['onStatsUpdate']>
+    >
+    /* Event triggered when slice transcription is stabilized */
+    onStabilized?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['RealtimeTranscription']['onStabilized']>
+    >
+    /* Event triggered when transcription ends */
+    onEnd?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['RealtimeTranscription']['onEnd']>
+    >
+  }
+  outlets?: {
+    /* Is realtime transcription currently active */
+    isActive?: () => Data<boolean>
+    /* Is currently transcribing audio */
+    isTranscribing?: () => Data<boolean>
+    /* Current transcription results */
+    results?: () => Data<
+      Array<{
+        transcribeEvent?: {
+          type?: string
+          sliceIndex?: number
+          isCapturing?: boolean
+          data?: {
+            result?: string
+            [key: string]: any
+          }
+          vadEvent?: {
+            type?: string
+            confidence?: number
+            duration?: number
+            sliceIndex?: number
+            timestamp?: number
+            [key: string]: any
+          }
+          [key: string]: any
+        }
+        vadEvent?: {
+          type?: string
+          confidence?: number
+          duration?: number
+          sliceIndex?: number
+          timestamp?: number
+          [key: string]: any
+        }
+        [key: string]: any
+      }>
+    >
+    /* Current transcription result text */
+    resultText?: () => Data<string>
+    /* Last stabilized transcription result segment */
+    lastStabilizedSegment?: () => Data<string>
+    /* Current statistics */
+    statistics?: () => Data<{
+      type?: string
+      timestamp?: number
+      data?: { [key: string]: any }
+      [key: string]: any
+    }>
+    /* Latest transcribe event */
+    lastTranscribeEvent?: () => Data<{
+      type?: string
+      sliceIndex?: number
+      isCapturing?: boolean
+      data?: {
+        result?: string
+        [key: string]: any
+      }
+      vadEvent?: {
+        type?: string
+        confidence?: number
+        duration?: number
+        sliceIndex?: number
+        timestamp?: number
+        [key: string]: any
+      }
+      [key: string]: any
+    }>
+    /* Latest VAD event */
+    lastVadEvent?: () => Data<{
+      type?: string
+      confidence?: number
+      duration?: number
+      sliceIndex?: number
+      timestamp?: number
+      [key: string]: any
+    }>
+    /* Stabilized transcription text from completed slices */
+    stabilizedText?: () => Data<string>
+    /* Audio output file path (auto-generated when saving audio) */
+    audioOutputPath?: () => Data<string>
+    /* Available microphone devices (Web / Desktop only) */
+    devices?: () => Data<Array<any>>
+  }
+}
+
+/* Realtime speech-to-text transcription using Whisper and VAD with live audio streaming */
+export type GeneratorRealtimeTranscription = Generator &
+  GeneratorRealtimeTranscriptionDef & {
+    templateKey: 'GENERATOR_REALTIME_TRANSCRIPTION'
+    switches?: Array<
+      SwitchDef &
+        GeneratorRealtimeTranscriptionDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'isActive'
+                    | 'isTranscribing'
+                    | 'results'
+                    | 'resultText'
+                    | 'lastStabilizedSegment'
+                    | 'statistics'
+                    | 'lastTranscribeEvent'
+                    | 'lastVadEvent'
+                    | 'stabilizedText'
+                    | 'audioOutputPath'
+                    | 'devices'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/RerankerGgml.ts

@@ -0,0 +1,191 @@
+/* Auto generated by build script
+ *
+ * Local rerank based on GGML and [llama.cpp](https://github.com/ggerganov/llama.cpp)
+ *
+ *  ## Notice
+ *  - The device RAM must be larger than 8GB
+ *  - iOS: Recommended use M1+ / A17+ chip device. Supported GPU acceleration by Metal.
+ *  - macOS: Recommended use M1+ chip device. Supported GPU acceleration by Metal.
+ *  - Android: Recommended use Android 13+ system.
+ *    - Supported GPU acceleration by OpenCL, currently only for Qualcomm Adreno 700+ GPUs, other GPUs are not supported.
+ *    - Supported Hexagon NPU for Qualcomm Snapdragon 8 Gen 2+ GPUs.
+ *  - Linux / Windows
+ *    - Supported GPU acceleration, you can choose `vulkan` or `cuda` backend in Accel Variant property
+ *    - Supported Hexagon NPU for Qualcomm Dragonwing IQ9 series+ (Linux)
+ */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+import type { TemplateEventPropsMap } from '../../utils/event-props'
+
+/* Load the model */
+export type GeneratorRerankerActionLoadModel = Action & {
+  __actionName: 'GENERATOR_RERANKER_LOAD_MODEL'
+}
+
+/* Rerank documents based on query relevance */
+export type GeneratorRerankerActionRerank = ActionWithParams & {
+  __actionName: 'GENERATOR_RERANKER_RERANK'
+  params?: Array<
+    | {
+        input: 'query'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'documents'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Release context */
+export type GeneratorRerankerActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_RERANKER_RELEASE_CONTEXT'
+}
+
+interface GeneratorRerankerDef {
+  /*
+Default property:
+{
+  "init": false,
+  "contextSize": 512,
+  "batchSize": 512,
+  "uBatchSize": 512,
+  "accelVariant": "default",
+  "mainGpu": 0,
+  "gpuLayers": 0,
+  "useMlock": true,
+  "useMmap": true,
+  "normalize": 1
+}
+  */
+  property?: {
+    /* Initialize the Reranker context on generator initialization */
+    init?: boolean | DataLink
+    /* The URL or path of reranker model (GGUF format) */
+    modelUrl?: string | DataLink
+    /* Hash of reranker model */
+    modelHash?: string | DataLink
+    /* Hash type of reranker model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Context size (0 ~ 4096) (Default to 512) */
+    contextSize?: number | DataLink
+    /* Logical batch size for processing (default: 512) */
+    batchSize?: number | DataLink
+    /* Physical maximum batch size (default: 512) */
+    uBatchSize?: number | DataLink
+    /* GGML accelerator variant (Only for desktop)
+ `default` - CPU / Metal (macOS)  
+ `vulkan` - Use Vulkan  
+ `cuda` - Use CUDA  
+ `snapdragon` - Use OpenCL/Hexagon of Qualcomm Snapdragon */
+    accelVariant?: 'default' | 'vulkan' | 'cuda' | 'snapdragon' | DataLink
+    /* Devices. For example:
+  
+ Metal or CPU for iOS/tvOS/MacOS  
+ OpenCL or CPU for Android  
+   - Add `HTP0`, `HTP1`, `...` for OpenCL/Hexagon devices (Use HTP* for all HTP devices)  
+ For Desktop, you may want to include/exclude GPUs */
+    devices?: Array<string | DataLink> | DataLink
+    /* Main GPU index (default: 0) */
+    mainGpu?: number | DataLink
+    /* Number of layers to store in VRAM (default: 0) */
+    gpuLayers?: number | DataLink
+    /* Maximum number of threads to use (default: auto) */
+    maxThreads?: number | DataLink
+    /* Use mlock to keep model in memory (default: true) */
+    useMlock?: boolean | DataLink
+    /* Use mmap for model loading (default: true) */
+    useMmap?: boolean | DataLink
+    /* Query text for reranking */
+    query?: string | DataLink
+    /* Array of documents to rerank */
+    documents?: Array<string | DataLink> | DataLink
+    /* Normalize reranking scores (default: from model config) */
+    normalize?: number | DataLink | boolean | DataLink | DataLink
+    /* Maximum number of documents to return (default: unlimited) */
+    topK?: number | DataLink
+  }
+  events?: {
+    /* Event triggered when the reranker context state changes (loading, ready, error, released) */
+    onContextStateChange?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['Reranker']['onContextStateChange']>
+    >
+    /* Event triggered when an error occurs during reranker operations */
+    onError?: Array<EventAction<string & keyof TemplateEventPropsMap['Reranker']['onError']>>
+  }
+  outlets?: {
+    /* Current state of the reranker context (loading, ready, error, released) */
+    contextState?: () => Data<string>
+    /* Loading progress of the reranker model (0-100) */
+    contextLoadProgress?: () => Data<number>
+    /* Detailed information about the reranker context including instance ID and processing status */
+    contextDetails?: () => Data<{
+      state?: string
+      contextId?: string
+      gpu?: string
+      reasonNoGPU?: string
+      [key: string]: any
+    }>
+    /* Result of the reranking operation containing scored and ranked documents */
+    rerankResult?: () => Data<
+      Array<{
+        index?: number
+        score?: number
+        text?: string
+        payload?: { [key: string]: any }
+        [key: string]: any
+      }>
+    >
+    /* Boolean indicating whether the reranker is currently processing a request */
+    isProcessing?: () => Data<boolean>
+  }
+}
+
+/* Local rerank based on GGML and [llama.cpp](https://github.com/ggerganov/llama.cpp)
+
+ ## Notice  
+ - The device RAM must be larger than 8GB  
+ - iOS: Recommended use M1+ / A17+ chip device. Supported GPU acceleration by Metal.  
+ - macOS: Recommended use M1+ chip device. Supported GPU acceleration by Metal.  
+ - Android: Recommended use Android 13+ system.  
+   - Supported GPU acceleration by OpenCL, currently only for Qualcomm Adreno 700+ GPUs, other GPUs are not supported.  
+   - Supported Hexagon NPU for Qualcomm Snapdragon 8 Gen 2+ GPUs.  
+ - Linux / Windows  
+   - Supported GPU acceleration, you can choose `vulkan` or `cuda` backend in Accel Variant property  
+   - Supported Hexagon NPU for Qualcomm Dragonwing IQ9 series+ (Linux) */
+export type GeneratorReranker = Generator &
+  GeneratorRerankerDef & {
+    templateKey: 'GENERATOR_RERANKER'
+    switches?: Array<
+      SwitchDef &
+        GeneratorRerankerDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'contextLoadProgress'
+                    | 'contextDetails'
+                    | 'rerankResult'
+                    | 'isProcessing'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/SerialPort.ts

@@ -0,0 +1,151 @@
+/* Auto generated by build script
+ *
+ * Serial port communication for USB/UART devices (Android, Desktop only)
+ */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+import type { TemplateEventPropsMap } from '../../utils/event-props'
+
+/* Get available serial devices */
+export type GeneratorSerialPortActionGeneratorSerialGetDeviceList = Action & {
+  __actionName: 'GENERATOR_SERIAL_GET_DEVICE_LIST'
+}
+
+/* Attach device */
+export type GeneratorSerialPortActionGeneratorSerialAttach = Action & {
+  __actionName: 'GENERATOR_SERIAL_ATTACH'
+}
+
+/* Dettach device */
+export type GeneratorSerialPortActionGeneratorSerialDetach = Action & {
+  __actionName: 'GENERATOR_SERIAL_DETACH'
+}
+
+/* Send string data (ASCII) */
+export type GeneratorSerialPortActionGeneratorSerialSendString = ActionWithParams & {
+  __actionName: 'GENERATOR_SERIAL_SEND_STRING'
+  params?: Array<
+    | {
+        input: 'generatorSerialStringValue'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'generatorSerialStringEncoding'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Send hex string (e.g. 0123456789ABCDEF) */
+export type GeneratorSerialPortActionGeneratorSerialSendHex = ActionWithParams & {
+  __actionName: 'GENERATOR_SERIAL_SEND_HEX'
+  params?: Array<{
+    input: 'generatorSerialHexValue'
+    value?: string | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+interface GeneratorSerialPortDef {
+  /*
+Default property:
+{
+  "attachOnInit": false,
+  "driver": "fd",
+  "parityBit": "none",
+  "dataBits": 8,
+  "stopBits": 1,
+  "commMode": "raw"
+}
+  */
+  property?: {
+    /* Try attach on generator initialized  On web require user activation to attach device */
+    attachOnInit?: boolean | DataLink
+    /* The serial device driver */
+    driver?: 'fd' | 'usb' | DataLink
+    /* The serial device path  e.g. /dev/ttyS0 or /dev/bus/usb/001/001  For desktop and web is device index number */
+    path?: string | DataLink
+    /* The serial USB vendor id (autoconnect first) */
+    vendorId?: number | DataLink
+    /* The serial USB product id */
+    productId?: number | DataLink
+    /* The super user binary path (For Android root permission) */
+    suPath?: string | DataLink
+    /* The serial device baud rate */
+    baudRate?: number | DataLink
+    /* The serial device parity bit */
+    parityBit?: 'none' | 'odd' | 'even' | DataLink
+    /* The serial device data bits */
+    dataBits?: number | DataLink
+    /* The serial device stop bits */
+    stopBits?: number | DataLink
+    /* The serial device communication mode */
+    commMode?: 'raw' | 'EDC' | 'Regex Match' | DataLink
+    /* The regex match pattern for packets */
+    regexMatch?: string | DataLink
+  }
+  events?: {
+    /* Event for attach the device */
+    generatorSerialOnAttach?: Array<EventAction>
+    /* Event for detach the device */
+    generatorSerialOnDetach?: Array<EventAction>
+    /* Event on error */
+    generatorSerialOnError?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['SerialPort']['generatorSerialOnError']>
+    >
+    /* Event for receive data */
+    generatorSerialOnData?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['SerialPort']['generatorSerialOnData']>
+    >
+  }
+  outlets?: {
+    /* Available serial devices */
+    generatorSerialDeviceList?: () => Data<Array<{ [key: string]: any }>>
+    /* Is serial open */
+    generatorSerialIsOpen?: () => Data<boolean>
+    /* Last rerceive data (Base64) */
+    generatorSerialLastDataBase64?: () => Data<string>
+    /* Last rerceive plain text */
+    generatorSerialLastDataString?: () => Data<string>
+    /* Error message */
+    generatorSerialErrorMessage?: () => Data<string>
+  }
+}
+
+/* Serial port communication for USB/UART devices (Android, Desktop only) */
+export type GeneratorSerialPort = Generator &
+  GeneratorSerialPortDef & {
+    templateKey: 'GENERATOR_SERIAL_PORT'
+    switches?: Array<
+      SwitchDef &
+        GeneratorSerialPortDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'generatorSerialDeviceList'
+                    | 'generatorSerialIsOpen'
+                    | 'generatorSerialLastDataBase64'
+                    | 'generatorSerialLastDataString'
+                    | 'generatorSerialErrorMessage'
+                  value: any
+                }
+          }>
+        }
+    >
+  }