@fugood/bricks-project 2.23.0-beta.9 → 2.23.2

package/types/generators/VadOnnx.ts

@@ -0,0 +1,201 @@
+/* Auto generated by build script */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Load the model */
+export type GeneratorVadInferenceOnnxActionLoadModel = Action & {
+  __actionName: 'GENERATOR_ONNX_VAD_LOAD_MODEL'
+}
+
+/* Detect speech in audio file */
+export type GeneratorVadInferenceOnnxActionDetectFile = ActionWithParams & {
+  __actionName: 'GENERATOR_ONNX_VAD_DETECT_FILE'
+  params?: Array<
+    | {
+        input: 'fileUrl'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSpeechDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSilenceDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxSpeechDurationS'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'speechPadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Detect speech in audio data */
+export type GeneratorVadInferenceOnnxActionDetectData = ActionWithParams & {
+  __actionName: 'GENERATOR_ONNX_VAD_DETECT_DATA'
+  params?: Array<
+    | {
+        input: 'data'
+        value?: any | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSpeechDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSilenceDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxSpeechDurationS'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'speechPadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Clean cache */
+export type GeneratorVadInferenceOnnxActionCleanCache = Action & {
+  __actionName: 'GENERATOR_ONNX_VAD_CLEAN_CACHE'
+}
+
+/* Release context */
+export type GeneratorVadInferenceOnnxActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_ONNX_VAD_RELEASE_CONTEXT'
+}
+
+interface GeneratorVadInferenceOnnxDef {
+  /*
+Default property:
+{
+  "modelType": "auto",
+  "detectThreshold": 0.5,
+  "detectMinSpeechDurationMs": 250,
+  "detectMinSilenceDurationMs": 100,
+  "detectMaxSpeechDurationS": 30,
+  "detectSpeechPadMs": 30,
+  "executionMode": "sequential"
+}
+  */
+  property?: {
+    /* Initialize the VAD context on generator initialization */
+    init?: boolean | DataLink
+    /* VAD model */
+    model?: string | DataLink
+    /* Model type (auto-detected from config.json) */
+    modelType?: string | DataLink
+    /* Quantize type */
+    quantizeType?:
+      | 'auto'
+      | 'none'
+      | 'fp16'
+      | 'q8'
+      | 'int8'
+      | 'uint8'
+      | 'q4'
+      | 'bnb4'
+      | 'q4f16'
+      | DataLink
+    /* Speech probability threshold (0.0-1.0) */
+    detectThreshold?: number | DataLink
+    /* Minimum speech duration in milliseconds */
+    detectMinSpeechDurationMs?: number | DataLink
+    /* Minimum silence duration in milliseconds */
+    detectMinSilenceDurationMs?: number | DataLink
+    /* Maximum speech duration in seconds */
+    detectMaxSpeechDurationS?: number | DataLink
+    /* Padding around speech segments in milliseconds */
+    detectSpeechPadMs?: number | DataLink
+    /* Executor candidates, descending order of priority  
+  Default will be xnnpack, wasm, cpu */
+    executors?:
+      | Array<'qnn' | 'dml' | 'nnapi' | 'xnnpack' | 'coreml' | 'cpu' | 'wasm' | 'webgpu' | DataLink>
+      | DataLink
+    /* Execution mode  
+  Usually when the model has many branches, setting this option to `parallel` will give you better performance. */
+    executionMode?: 'sequential' | 'parallel' | DataLink
+    /* QNN backend */
+    qnnBackend?: 'HTP' | 'HTA' | 'DSP' | 'GPU' | 'CPU' | DataLink
+    /* Enable FP16 for QNN HTP */
+    qnnHtpEnableFp16?: boolean | DataLink
+    /* Enable QNN debug */
+    qnnEnableDebug?: boolean | DataLink
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+    /* Event triggered when got detection result */
+    onDetected?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Is detecting */
+    isDetecting?: () => Data
+    /* Detection segments result */
+    detectionSegments?: () => Data
+    /* Detection details */
+    detectionDetails?: () => Data
+  }
+}
+
+/* Local Voice Activity Detection (VAD) inference based on [transformers.js](https://huggingface.co/docs/transformers.js)  
+ You can use any compatible VAD model from HuggingFace (Silero VAD, smart-turn, etc.) */
+export type GeneratorVadInferenceOnnx = Generator &
+  GeneratorVadInferenceOnnxDef & {
+    templateKey: 'GENERATOR_ONNX_VAD'
+    switches: Array<
+      SwitchDef &
+        GeneratorVadInferenceOnnxDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'contextState' | 'isDetecting' | 'detectionSegments' | 'detectionDetails'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/VadTraditional.ts

@@ -0,0 +1,123 @@
+/* Auto generated by build script */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Detect speech in audio file */
+export type GeneratorVadInferenceTraditionalActionDetectFile = ActionWithParams & {
+  __actionName: 'GENERATOR_TRADITIONAL_VAD_DETECT_FILE'
+  params?: Array<{
+    input: 'fileUrl'
+    value?: string | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Detect speech in audio data stream */
+export type GeneratorVadInferenceTraditionalActionDetectData = ActionWithParams & {
+  __actionName: 'GENERATOR_TRADITIONAL_VAD_DETECT_DATA'
+  params?: Array<{
+    input: 'data'
+    value?: any | EventProperty
+    mapping?: string
+  }>
+}
+
+interface GeneratorVadInferenceTraditionalDef {
+  /*
+Default property:
+{
+  "detectVocalFreqMin": 75,
+  "detectVocalFreqMax": 900,
+  "detectThreshold": 0.5,
+  "detectMinSpeechDurationMs": 250,
+  "detectMinSilenceDurationMs": 100,
+  "detectMaxSpeechDurationS": 30,
+  "detectSpeechPadMs": 30
+}
+  */
+  property?: {
+    /* Minimum vocal frequency in Hz */
+    detectVocalFreqMin?: number | DataLink
+    /* Maximum vocal frequency in Hz */
+    detectVocalFreqMax?: number | DataLink
+    /* Volume threshold in dB */
+    detectVolumeThreshold?: number | DataLink
+    /* Speech probability threshold (0.0-1.0) - maps to frequency clarity */
+    detectThreshold?: number | DataLink
+    /* Minimum speech duration in milliseconds */
+    detectMinSpeechDurationMs?: number | DataLink
+    /* Minimum silence duration in milliseconds */
+    detectMinSilenceDurationMs?: number | DataLink
+    /* Maximum speech duration in seconds */
+    detectMaxSpeechDurationS?: number | DataLink
+    /* Padding around speech segments in milliseconds */
+    detectSpeechPadMs?: number | DataLink
+    /* The file URL or path to be analyzed */
+    detectFileUrl?: string | DataLink
+    /* MD5 of file to be analyzed */
+    detectFileMd5?: string | DataLink
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when detection result is available */
+    onDetected?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Is detecting */
+    isDetecting?: () => Data
+    /* Is speaking (real-time) */
+    isSpeaking?: () => Data
+    /* Detection segments result */
+    detectionSegments?: () => Data
+    /* Current volume in dB */
+    currentVolume?: () => Data
+    /* Current frequency clarity (0-1) */
+    currentClarity?: () => Data
+    /* Current detected frequency in Hz */
+    currentFrequency?: () => Data
+  }
+}
+
+/* Traditional Voice Activity Detection (VAD) using pitch detection and RMS volume analysis  
+ No model download required - pure algorithmic approach */
+export type GeneratorVadInferenceTraditional = Generator &
+  GeneratorVadInferenceTraditionalDef & {
+    templateKey: 'GENERATOR_TRADITIONAL_VAD'
+    switches: Array<
+      SwitchDef &
+        GeneratorVadInferenceTraditionalDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'isDetecting'
+                    | 'isSpeaking'
+                    | 'detectionSegments'
+                    | 'currentVolume'
+                    | 'currentClarity'
+                    | 'currentFrequency'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/VectorStore.ts

@@ -1,6 +1,8 @@
+/* Auto generated by build script */
 import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
 import type { Data, DataLink } from '../data'
 import type {
+  Brick,
   Generator,
   EventAction,
   ActionWithDataParams,
@@ -159,8 +161,19 @@ Default property:
     ggmlContextSize?: number | DataLink
     /* Pooling type of ggml model */
     ggmlPoolingType?: 'none' | 'mean' | 'cls' | 'last' | 'rank' | DataLink
-    /* Accelerator variant of ggml model */
-    ggmlAccelVariant?: 'default' | 'vulkan' | 'cuda' | DataLink
+    /* GGML accelerator variant (Only for desktop)  
+ `default` - CPU / Metal (macOS)  
+ `vulkan` - Use Vulkan  
+ `cuda` - Use CUDA  
+ `snapdragon` - Use OpenCL/Hexagon of Qualcomm Snapdragon */
+    ggmlAccelVariant?: 'default' | 'vulkan' | 'cuda' | 'snapdragon' | DataLink
+    /* Devices. For example:  
+  
+ Metal or CPU for iOS/tvOS/MacOS  
+ OpenCL or CPU for Android  
+   - Add `HTP0`, `HTP1`, `...` for OpenCL/Hexagon devices (Use HTP* for all HTP devices)  
+ For Desktop, you may want to include/exclude GPUs */
+    ggmlDevices?: Array<string | DataLink> | DataLink
     /* Normalize */
     ggmlEmbdNormalize?: number | DataLink
     /* API key of OpenAI Compatible API */

package/types/generators/Watchdog.ts

@@ -1,6 +1,8 @@
+/* Auto generated by build script */
 import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
 import type { Data, DataLink } from '../data'
 import type {
+  Brick,
   Generator,
   EventAction,
   ActionWithDataParams,

package/types/generators/WebCrawler.ts

@@ -1,6 +1,8 @@
+/* Auto generated by build script */
 import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
 import type { Data, DataLink } from '../data'
 import type {
+  Brick,
   Generator,
   EventAction,
   ActionWithDataParams,

package/types/generators/WebRtc.ts

@@ -1,6 +1,8 @@
+/* Auto generated by build script */
 import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
 import type { Data, DataLink } from '../data'
 import type {
+  Brick,
   Generator,
   EventAction,
   ActionWithDataParams,
@@ -58,9 +60,9 @@ Default property:
     /* Calee ID, to ilter caller signal */
     calleeId?: string | DataLink
     /* Target local WebRTC stream preview brick ID */
-    localVideoTarget?: string | DataLink
+    localVideoTarget?: string | DataLink | (() => Brick)
     /* Target remote WebRTC stream brick ID */
-    remoteVideoTarget?: string | DataLink
+    remoteVideoTarget?: string | DataLink | (() => Brick)
     /* ICE Server list  
   Default use Google public STUN servers if not setted. */
     iceServers?:

package/types/generators/WebSocket.ts

@@ -1,6 +1,8 @@
+/* Auto generated by build script */
 import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
 import type { Data, DataLink } from '../data'
 import type {
+  Brick,
   Generator,
   EventAction,
   ActionWithDataParams,

package/types/generators/index.ts

@@ -33,9 +33,11 @@ export * from './ThermalPrinter'
 export * from './SqLite'
 export * from './McpServer'
 export * from './Mcp'
+export * from './VadTraditional'
 export * from './TextToSpeechOnnx'
 export * from './LlmOnnx'
 export * from './SpeechToTextOnnx'
+export * from './VadOnnx'
 export * from './SpeechToTextGgml'
 export * from './VadGgml'
 export * from './RealtimeTranscription'
@@ -46,5 +48,8 @@ export * from './LlmQualcommAiEngine'
 export * from './LlmOpenAiCompat'
 export * from './TextToSpeechOpenAiLike'
 export * from './LlmAnthropicCompat'
+export * from './LlmAppleBuiltin'
+export * from './SttAppleBuiltin'
+export * from './TextToSpeechAppleBuiltin'
 export * from './Assistant'
 export * from './VectorStore'

package/types/index.ts

@@ -1,5 +1,7 @@
 export * from './data'
 export * from './data-calc'
+export * from './data-calc-script'
+export * from './data-calc-command'
 export * from './generators'
 export * from './bricks'
 export * from './animation'
@@ -8,3 +10,4 @@ export * from './subspace'
 export * from './common'
 export * from './switch'
 export * from './system'
+export * from './automation'

package/types/system.ts

@@ -1,3 +1,4 @@
+/* Auto generated by build script */
 import type {
   Action,
   ActionWithDataParams,
@@ -10,7 +11,7 @@ import type { Animation } from './animation'
 import type { Canvas } from './canvas'
 import type { Data, DataLink } from './data'
 
-/* Change Data value with script expression */
+/* Change Data value with script expression. One expression of return value */
 export type SystemActionPropertyBankExpression = ActionWithParams & {
   __actionName: 'PROPERTY_BANK_EXPRESSION'
   params?: Array<
@@ -21,7 +22,13 @@ export type SystemActionPropertyBankExpression = ActionWithParams & {
       }
     | {
         input: 'variables'
-        value?: {} | DataLink | EventProperty
+        value?:
+          | Array<{
+              id?: string
+              key?: string
+            }>
+          | DataLink
+          | EventProperty
         mapping?: string
       }
     | {
@@ -152,7 +159,13 @@ export type SystemActionChangeCanvas = ActionWithParams & {
       }
     | {
         input: 'canvasKeyMapList'
-        value?: {} | DataLink | EventProperty
+        value?:
+          | Array<{
+              key?: string
+              value?: string
+            }>
+          | DataLink
+          | EventProperty
         mapping?: string
       }
     | {
@@ -217,7 +230,15 @@ export type SystemActionAlert = ActionWithParams & {
       }
     | {
         input: 'selections'
-        value?: {} | DataLink | EventProperty
+        value?:
+          | Array<{
+              label?: string
+              color?: string
+              value?: string
+              disableResult?: boolean
+            }>
+          | DataLink
+          | EventProperty
         mapping?: string
       }
     | {
@@ -414,7 +435,15 @@ export type SystemActionChannelSubscribe = ActionWithParams & {
       }
     | {
         input: 'dataAssignList'
-        value?: {} | DataLink | EventProperty
+        value?:
+          | Array<{
+              path?: string
+              type?: any
+              id?: string
+              required?: boolean
+            }>
+          | DataLink
+          | EventProperty
         mapping?: string
       }
     | {
@@ -424,7 +453,15 @@ export type SystemActionChannelSubscribe = ActionWithParams & {
       }
     | {
         input: 'progressAssignList'
-        value?: {} | DataLink | EventProperty
+        value?:
+          | Array<{
+              path?: string
+              type?: any
+              id?: string
+              required?: boolean
+            }>
+          | DataLink
+          | EventProperty
         mapping?: string
       }
     | {
@@ -605,6 +642,11 @@ export type SystemActionThrowException = ActionWithParams & {
   >
 }
 
+/* Restart the foundation (React Native bundle) */
+export type SystemActionRestartFoundation = Action & {
+  __actionName: 'RESTART_FOUNDATION'
+}
+
 /* Dismiss keyboard */
 export type SystemActionDismissKeyboard = Action & {
   __actionName: 'DISMISS_KEYBOARD'

package/utils/calc.ts

@@ -1,4 +1,8 @@
-import type { DataCalculationMap, DataCalculationData, DataCommand } from '../types/data-calc'
+import type {
+  DataCommand,
+  DataCalculationMap,
+  DataCalculationData,
+} from '../types/data-calc-command'
 
 const GRID = {
   WIDTH: 300, // Distance between columns
@@ -29,10 +33,11 @@ export const generateDataCalculationMapEditorInfo = (
   nodes.forEach((node) => {
     // Count and track inputs
     if ('inputs' in node) {
-      const inputs = node.inputs
-        .filter((input) => input !== null)
-        .map((input) => (Array.isArray(input) ? input.length : 1))
-        .reduce((sum, count) => sum + count, 0)
+      const inputs = node.inputs.reduce((count, input) => {
+        if (input === null) return count
+        if (Array.isArray(input)) return count + input.length
+        return count + 1
+      }, 0)
       inputCounts.set(node, inputs)
 
       // Track connections
@@ -55,10 +60,11 @@ export const generateDataCalculationMapEditorInfo = (
 
     // Count outputs
     if ('outputs' in node) {
-      const outputs = node.outputs
-        .filter((output) => output !== null)
-        .map((output) => (Array.isArray(output) ? output.length : 1))
-        .reduce((sum, count) => sum + count, 0)
+      const outputs = node.outputs.reduce((count, output) => {
+        if (output === null) return count
+        if (Array.isArray(output)) return count + output.length
+        return count + 1
+      }, 0)
       outputCounts.set(node, outputs)
     } else {
       outputCounts.set(node, 0)

package/utils/data.ts

@@ -1,3 +1,4 @@
+/* Auto generated by build script */
 import type { DataLink, Data, Canvas } from '../types'
 import { makeId } from './id'