@fugood/bricks-ctor 2.24.0-beta.40

package/types/generators/VadOnnx.ts

@@ -0,0 +1,231 @@
+/* Auto generated by build script
+ *
+ * Local Voice Activity Detection (VAD) inference based on [transformers.js](https://huggingface.co/docs/transformers.js)
+ *  You can use any compatible VAD model from HuggingFace (Silero VAD, smart-turn, etc.)
+ */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+import type { TemplateEventPropsMap } from '../../utils/event-props'
+
+/* Load the model */
+export type GeneratorVadInferenceOnnxActionLoadModel = Action & {
+  __actionName: 'GENERATOR_ONNX_VAD_LOAD_MODEL'
+}
+
+/* Detect speech in audio file */
+export type GeneratorVadInferenceOnnxActionDetectFile = ActionWithParams & {
+  __actionName: 'GENERATOR_ONNX_VAD_DETECT_FILE'
+  params?: Array<
+    | {
+        input: 'fileUrl'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSpeechDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSilenceDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxSpeechDurationS'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'speechPadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Detect speech in audio data */
+export type GeneratorVadInferenceOnnxActionDetectData = ActionWithParams & {
+  __actionName: 'GENERATOR_ONNX_VAD_DETECT_DATA'
+  params?: Array<
+    | {
+        input: 'data'
+        value?: any | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSpeechDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSilenceDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxSpeechDurationS'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'speechPadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Clean cache */
+export type GeneratorVadInferenceOnnxActionCleanCache = Action & {
+  __actionName: 'GENERATOR_ONNX_VAD_CLEAN_CACHE'
+}
+
+/* Release context */
+export type GeneratorVadInferenceOnnxActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_ONNX_VAD_RELEASE_CONTEXT'
+}
+
+interface GeneratorVadInferenceOnnxDef {
+  /*
+Default property:
+{
+  "modelType": "auto",
+  "detectThreshold": 0.5,
+  "detectMinSpeechDurationMs": 250,
+  "detectMinSilenceDurationMs": 100,
+  "detectMaxSpeechDurationS": 30,
+  "detectSpeechPadMs": 30,
+  "executionMode": "sequential"
+}
+  */
+  property?: {
+    /* Initialize the VAD context on generator initialization */
+    init?: boolean | DataLink
+    /* VAD model */
+    model?: string | DataLink
+    /* Model type (auto-detected from config.json) */
+    modelType?: string | DataLink
+    /* Quantize type */
+    quantizeType?:
+      | 'auto'
+      | 'none'
+      | 'fp16'
+      | 'q8'
+      | 'int8'
+      | 'uint8'
+      | 'q4'
+      | 'bnb4'
+      | 'q4f16'
+      | DataLink
+    /* Speech probability threshold (0.0-1.0) */
+    detectThreshold?: number | DataLink
+    /* Minimum speech duration in milliseconds */
+    detectMinSpeechDurationMs?: number | DataLink
+    /* Minimum silence duration in milliseconds */
+    detectMinSilenceDurationMs?: number | DataLink
+    /* Maximum speech duration in seconds */
+    detectMaxSpeechDurationS?: number | DataLink
+    /* Padding around speech segments in milliseconds */
+    detectSpeechPadMs?: number | DataLink
+    /* Executor candidates, descending order of priority
+  Default will be xnnpack, wasm, cpu */
+    executors?:
+      | Array<'qnn' | 'dml' | 'nnapi' | 'xnnpack' | 'coreml' | 'cpu' | 'wasm' | 'webgpu' | DataLink>
+      | DataLink
+    /* Execution mode
+  Usually when the model has many branches, setting this option to `parallel` will give you better performance. */
+    executionMode?: 'sequential' | 'parallel' | DataLink
+    /* QNN backend */
+    qnnBackend?: 'HTP' | 'HTA' | 'DSP' | 'GPU' | 'CPU' | DataLink
+    /* Enable FP16 for QNN HTP */
+    qnnHtpEnableFp16?: boolean | DataLink
+    /* Enable QNN debug */
+    qnnEnableDebug?: boolean | DataLink
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['OnnxVad']['onContextStateChange']>
+    >
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction<string & keyof TemplateEventPropsMap['OnnxVad']['onError']>>
+    /* Event triggered when got detection result */
+    onDetected?: Array<EventAction<string & keyof TemplateEventPropsMap['OnnxVad']['onDetected']>>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data<string>
+    /* Is detecting */
+    isDetecting?: () => Data<boolean>
+    /* Detection segments result */
+    detectionSegments?: () => Data<
+      Array<{
+        start?: number
+        end?: number
+        confidence?: number
+        t0?: number
+        t1?: number
+        [key: string]: any
+      }>
+    >
+    /* Detection details */
+    detectionDetails?: () => Data<{
+      result?: string
+      segments?: Array<{
+        start?: number
+        end?: number
+        confidence?: number
+        t0?: number
+        t1?: number
+        [key: string]: any
+      }>
+      time?: number
+      startTime?: number
+      endTime?: number
+      [key: string]: any
+    }>
+  }
+}
+
+/* Local Voice Activity Detection (VAD) inference based on [transformers.js](https://huggingface.co/docs/transformers.js)
+ You can use any compatible VAD model from HuggingFace (Silero VAD, smart-turn, etc.) */
+export type GeneratorVadInferenceOnnx = Generator &
+  GeneratorVadInferenceOnnxDef & {
+    templateKey: 'GENERATOR_ONNX_VAD'
+    switches?: Array<
+      SwitchDef &
+        GeneratorVadInferenceOnnxDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'contextState' | 'isDetecting' | 'detectionSegments' | 'detectionDetails'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/VadTraditional.ts

@@ -0,0 +1,138 @@
+/* Auto generated by build script
+ *
+ * Traditional Voice Activity Detection (VAD) using pitch detection and RMS volume analysis
+ *  No model download required - pure algorithmic approach
+ */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+import type { TemplateEventPropsMap } from '../../utils/event-props'
+
+/* Detect speech in audio file */
+export type GeneratorVadInferenceTraditionalActionDetectFile = ActionWithParams & {
+  __actionName: 'GENERATOR_TRADITIONAL_VAD_DETECT_FILE'
+  params?: Array<{
+    input: 'fileUrl'
+    value?: string | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Detect speech in audio data stream */
+export type GeneratorVadInferenceTraditionalActionDetectData = ActionWithParams & {
+  __actionName: 'GENERATOR_TRADITIONAL_VAD_DETECT_DATA'
+  params?: Array<{
+    input: 'data'
+    value?: any | EventProperty
+    mapping?: string
+  }>
+}
+
+interface GeneratorVadInferenceTraditionalDef {
+  /*
+Default property:
+{
+  "detectVocalFreqMin": 75,
+  "detectVocalFreqMax": 900,
+  "detectThreshold": 0.5,
+  "detectMinSpeechDurationMs": 250,
+  "detectMinSilenceDurationMs": 100,
+  "detectMaxSpeechDurationS": 30,
+  "detectSpeechPadMs": 30
+}
+  */
+  property?: {
+    /* Minimum vocal frequency in Hz */
+    detectVocalFreqMin?: number | DataLink
+    /* Maximum vocal frequency in Hz */
+    detectVocalFreqMax?: number | DataLink
+    /* Volume threshold in dB */
+    detectVolumeThreshold?: number | DataLink
+    /* Speech probability threshold (0.0-1.0) - maps to frequency clarity */
+    detectThreshold?: number | DataLink
+    /* Minimum speech duration in milliseconds */
+    detectMinSpeechDurationMs?: number | DataLink
+    /* Minimum silence duration in milliseconds */
+    detectMinSilenceDurationMs?: number | DataLink
+    /* Maximum speech duration in seconds */
+    detectMaxSpeechDurationS?: number | DataLink
+    /* Padding around speech segments in milliseconds */
+    detectSpeechPadMs?: number | DataLink
+    /* The file URL or path to be analyzed */
+    detectFileUrl?: string | DataLink
+    /* MD5 of file to be analyzed */
+    detectFileMd5?: string | DataLink
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['TraditionalVad']['onContextStateChange']>
+    >
+    /* Event triggered when detection result is available */
+    onDetected?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['TraditionalVad']['onDetected']>
+    >
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction<string & keyof TemplateEventPropsMap['TraditionalVad']['onError']>>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data<string>
+    /* Is detecting */
+    isDetecting?: () => Data<boolean>
+    /* Is speaking (real-time) */
+    isSpeaking?: () => Data<boolean>
+    /* Detection segments result */
+    detectionSegments?: () => Data<
+      Array<{
+        t0?: number
+        t1?: number
+        [key: string]: any
+      }>
+    >
+    /* Current volume in dB */
+    currentVolume?: () => Data<number>
+    /* Current frequency clarity (0-1) */
+    currentClarity?: () => Data<number>
+    /* Current detected frequency in Hz */
+    currentFrequency?: () => Data<number>
+  }
+}
+
+/* Traditional Voice Activity Detection (VAD) using pitch detection and RMS volume analysis
+ No model download required - pure algorithmic approach */
+export type GeneratorVadInferenceTraditional = Generator &
+  GeneratorVadInferenceTraditionalDef & {
+    templateKey: 'GENERATOR_TRADITIONAL_VAD'
+    switches?: Array<
+      SwitchDef &
+        GeneratorVadInferenceTraditionalDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'isDetecting'
+                    | 'isSpeaking'
+                    | 'detectionSegments'
+                    | 'currentVolume'
+                    | 'currentClarity'
+                    | 'currentFrequency'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/VectorStore.ts

@@ -0,0 +1,257 @@
+/* Auto generated by build script
+ *
+ * Vector database for semantic search. Supports GGML (on-device GGUF models) and OpenAI Compatible API (OpenAI/ollama/llama.cpp) as embedding sources
+ */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+import type { TemplateEventPropsMap } from '../../utils/event-props'
+
+/* Load the model (Only if source is ggml) */
+export type GeneratorVectorStoreActionModelLoad = Action & {
+  __actionName: 'GENERATOR_VECTOR_STORE_MODEL_LOAD'
+}
+
+/* Release the model (Only if source is ggml) */
+export type GeneratorVectorStoreActionModelRelease = Action & {
+  __actionName: 'GENERATOR_VECTOR_STORE_MODEL_RELEASE'
+}
+
+/* Reset the embedding session */
+export type GeneratorVectorStoreActionReset = ActionWithParams & {
+  __actionName: 'GENERATOR_VECTOR_STORE_RESET'
+  params?: Array<{
+    input: 'resetType'
+    value?: 'session' | 'all' | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Insert file content with path or url, support Office / OpenOffice / PDF
+
+ PDF is not currently supprted in iOS / Android */
+export type GeneratorVectorStoreActionInsertFile = ActionWithParams & {
+  __actionName: 'GENERATOR_VECTOR_STORE_INSERT_FILE'
+  params?: Array<
+    | {
+        input: 'filePath'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileExtension'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'chunkSize'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Cancel the file download */
+export type GeneratorVectorStoreActionCancelFileDownload = Action & {
+  __actionName: 'GENERATOR_VECTOR_STORE_CANCEL_FILE_DOWNLOAD'
+}
+
+/* Insert pure text content to vector store */
+export type GeneratorVectorStoreActionInsertContent = ActionWithParams & {
+  __actionName: 'GENERATOR_VECTOR_STORE_INSERT_CONTENT'
+  params?: Array<
+    | {
+        input: 'textContent'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'chunkSize'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Remove file from vector store */
+export type GeneratorVectorStoreActionRemoveFile = ActionWithParams & {
+  __actionName: 'GENERATOR_VECTOR_STORE_REMOVE_FILE'
+  params?: Array<{
+    input: 'fileName'
+    value?: string | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Search similar text chunks in vector store */
+export type GeneratorVectorStoreActionSearch = ActionWithParams & {
+  __actionName: 'GENERATOR_VECTOR_STORE_SEARCH'
+  params?: Array<
+    | {
+        input: 'text'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'filename'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'limit'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'ignoreThreshold'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+interface GeneratorVectorStoreDef {
+  /*
+Default property:
+{
+  "source": "ggml",
+  "ggmlContextSize": 512,
+  "openaiBaseUrl": "https://api.openai.com/v1",
+  "openaiModel": "text-embedding-3-small",
+  "embedSessionId": "default-embeddings",
+  "fileChunkSize": 256
+}
+  */
+  property?: {
+    /* Initialize embedding model context & tokenizer context on subspace mounted */
+    init?: boolean | DataLink
+    /* Source of embedding model. Note that you can't use the saved embedddings data in different sources. */
+    source?: 'ggml' | 'openai' | DataLink
+    /* Path of ggml model */
+    ggmlModelUrl?: string | DataLink
+    /* Hash type of ggml model */
+    ggmlModelHashType?: 'sha256' | 'sha1' | 'md5' | DataLink
+    /* Hash of ggml model */
+    ggmlModelHash?: string | DataLink
+    /* Size of ggml context */
+    ggmlContextSize?: number | DataLink
+    /* Pooling type of ggml model */
+    ggmlPoolingType?: 'none' | 'mean' | 'cls' | 'last' | 'rank' | DataLink
+    /* GGML accelerator variant (Only for desktop)
+ `default` - CPU / Metal (macOS)  
+ `vulkan` - Use Vulkan  
+ `cuda` - Use CUDA  
+ `snapdragon` - Use OpenCL/Hexagon of Qualcomm Snapdragon */
+    ggmlAccelVariant?: 'default' | 'vulkan' | 'cuda' | 'snapdragon' | DataLink
+    /* Devices. For example:
+  
+ Metal or CPU for iOS/tvOS/MacOS  
+ OpenCL or CPU for Android  
+   - Add `HTP0`, `HTP1`, `...` for OpenCL/Hexagon devices (Use HTP* for all HTP devices)  
+ For Desktop, you may want to include/exclude GPUs */
+    ggmlDevices?: Array<string | DataLink> | DataLink
+    /* Normalize */
+    ggmlEmbdNormalize?: number | DataLink
+    /* API key of OpenAI Compatible API */
+    openaiApiKey?: string | DataLink
+    /* Base URL of OpenAI Compatible API. For example, we can use ollama / llama.cpp server instead of openai */
+    openaiBaseUrl?: string | DataLink
+    /* Model of OpenAI Compatible API */
+    openaiModel?: 'text-embedding-3-small' | 'text-embedding-3-large' | DataLink
+    /* Session ID of embedding */
+    embedSessionId?: string | DataLink
+    /* Storage type of embedding */
+    embedStorageType?: 'file' | 'memory' | DataLink
+    /* Tokenizer model URL (Recommended to use same with your LLM model, the model will just load with vocab_only mode) */
+    tokenizerModelUrl?: string | DataLink
+    /* Tokenizer model hash */
+    tokenizerModelHash?: string | DataLink
+    /* Tokenizer model hash type */
+    tokenizerModelHashType?: 'sha256' | 'sha1' | 'md5' | DataLink
+    /* Chunk size to handle file content (token), must be less then embedding model context size */
+    fileChunkSize?: number | DataLink
+  }
+  events?: {
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction<string & keyof TemplateEventPropsMap['VectorStore']['onError']>>
+  }
+  outlets?: {
+    /* undefined */
+    isReady?: () => Data<boolean>
+    /* Whether the embedding is processing */
+    processing?: () => Data<boolean>
+    /* Progress of embedding processing */
+    processProgress?: () => Data<{
+      chunking?: number
+      embedding?: number
+      writing?: number
+      [key: string]: any
+    }>
+    /* Files inserted to embedding */
+    files?: () => Data<Array<string>>
+    /* Search result */
+    searchResult?: () => Data<
+      Array<{
+        sessionId?: string
+        payload?: { [key: string]: any }
+        filename?: string
+        text?: string
+        textTokenSize?: number
+        fileTokenPos?: number
+        fileTokenLength?: number
+        distance?: number
+        [key: string]: any
+      }>
+    >
+  }
+}
+
+/* Vector database for semantic search. Supports GGML (on-device GGUF models) and OpenAI Compatible API (OpenAI/ollama/llama.cpp) as embedding sources */
+export type GeneratorVectorStore = Generator &
+  GeneratorVectorStoreDef & {
+    templateKey: 'GENERATOR_VECTOR_STORE'
+    switches?: Array<
+      SwitchDef &
+        GeneratorVectorStoreDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'isReady' | 'processing' | 'processProgress' | 'files' | 'searchResult'
+                  value: any
+                }
+          }>
+        }
+    >
+  }