@fugood/bricks-project 2.22.0-beta.9 → 2.22.1

package/types/generators/Tick.ts

@@ -0,0 +1,75 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Start the tick */
+export type GeneratorTickAction_ = Action & {
+  __actionName: 'GENERATOR_TICK'
+}
+
+/* Cancel the tick */
+export type GeneratorTickActionCancel = Action & {
+  __actionName: 'GENERATOR_TICK_CANCEL'
+}
+
+interface GeneratorTickDef {
+  /*
+Default property:
+{}
+  */
+  property?: {
+    /* Start tick on generator initialized */
+    init?: boolean | DataLink
+    /* Interval of second for countdown */
+    interval?: number | DataLink
+    /* Initial value of countdown */
+    countdownStartValue?: number | DataLink
+    /* Interval of value for countdown */
+    countdownInterval?: number | DataLink
+    /* Make countdown repeatable */
+    repeat?: boolean | DataLink
+    /* Map for PROPERTY_BANK_RESULT */
+    map?: {} | DataLink
+    /* Fallback of MAP_OBJECT */
+    mapFallback?: any
+  }
+  events?: {
+    /* Event for each tick start */
+    ticking?: Array<EventAction>
+    /* Event for tick completed */
+    completed?: Array<EventAction>
+  }
+  outlets?: {
+    /* Countdown step value */
+    countdown?: () => Data
+  }
+}
+
+/* Execute event in countdown manner */
+export type GeneratorTick = Generator &
+  GeneratorTickDef & {
+    templateKey: 'GENERATOR_TICK'
+    switches: Array<
+      SwitchDef &
+        GeneratorTickDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'countdown'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/Udp.ts

@@ -0,0 +1,109 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Bind UDP port */
+export type GeneratorUDPActionBind = Action & {
+  __actionName: 'GENERATOR_UDP_BIND'
+}
+
+/* Send data to target */
+export type GeneratorUDPActionSend = ActionWithParams & {
+  __actionName: 'GENERATOR_UDP_SEND'
+  params?: Array<
+    | {
+        input: 'targetHost'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'targetPort'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'sendData'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Close UDP */
+export type GeneratorUDPActionClose = Action & {
+  __actionName: 'GENERATOR_UDP_CLOSE'
+}
+
+interface GeneratorUDPDef {
+  /*
+Default property:
+{
+  "init": false,
+  "type": "udp4",
+  "encoding": "base64",
+  "broadcast": false,
+  "multicast": ""
+}
+  */
+  property?: {
+    /* Bind port on generator initialized */
+    init?: boolean | DataLink
+    /* The family of socket */
+    type?: 'udp4' | 'udp6' | DataLink
+    /* Encoding of data */
+    encoding?: 'base64' | 'utf8' | 'ascii' | DataLink
+    /* Local bind port of UDP (0 or empty is random) */
+    localPort?: number | DataLink
+    /* Enable receive broadcast packets */
+    broadcast?: boolean | DataLink
+    /* Join multicast group (Multicast address) */
+    multicast?: string | DataLink
+  }
+  events?: {
+    /* Event of UDP port is binded */
+    onReady?: Array<EventAction>
+    /* Event of receive data */
+    onData?: Array<EventAction>
+    /* Event of socket closeed */
+    onClose?: Array<EventAction>
+    /* Event of socket error */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* Local UDP binded port */
+    port?: () => Data
+    /* Last received packet */
+    lastReceive?: () => Data
+    /* Error message */
+    errorMessage?: () => Data
+  }
+}
+
+/* TCP Server */
+export type GeneratorUDP = Generator &
+  GeneratorUDPDef & {
+    templateKey: 'GENERATOR_UDP'
+    switches: Array<
+      SwitchDef &
+        GeneratorUDPDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'port' | 'lastReceive' | 'errorMessage'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/VadGgml.ts

@@ -0,0 +1,211 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Load the model */
+export type GeneratorVadInferenceActionLoadModel = Action & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_LOAD_MODEL'
+}
+
+/* Detect speech in audio file. You can provide `File URL` property, if not provided, it will use the default `File URL` */
+export type GeneratorVadInferenceActionDetectFile = ActionWithParams & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_DETECT_FILE'
+  params?: Array<
+    | {
+        input: 'fileUrl'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSpeechDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSilenceDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxSpeechDurationS'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'speechPadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'samplesOverlap'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Detect speech in audio data. Currently only support base64 encoded audio data (16-bit PCM, mono, 16kHz) */
+export type GeneratorVadInferenceActionDetectData = ActionWithParams & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_DETECT_DATA'
+  params?: Array<
+    | {
+        input: 'data'
+        value?: any | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSpeechDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'minSilenceDurationMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxSpeechDurationS'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'speechPadMs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'samplesOverlap'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Clear downloaded files (model, audio) & current jobs */
+export type GeneratorVadInferenceActionClearDownload = Action & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_CLEAR_DOWNLOAD'
+}
+
+/* Release context */
+export type GeneratorVadInferenceActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_VAD_INFERENCE_RELEASE_CONTEXT'
+}
+
+interface GeneratorVadInferenceDef {
+  /*
+Default property:
+{
+  "init": false,
+  "modelName": "silero-v5.1.2",
+  "modelUseGPU": true,
+  "modelThreads": 4,
+  "detectThreshold": 0.5,
+  "detectMinSpeechDurationMs": 250,
+  "detectMinSilenceDurationMs": 100,
+  "detectMaxSpeechDurationS": 30,
+  "detectSpeechPadMs": 30,
+  "detectSamplesOverlap": 0.1
+}
+  */
+  property?: {
+    /* Initialize the VAD context on generator initialization  
+ Please note that it will take some RAM depending on the model size */
+    init?: boolean | DataLink
+    /* Use model name, currently only supports the Silero VAD model.  
+ The model download progress will be done in preload stage or  the generator initialization stage.  
+ You can also choose `custom` option and set `Model URL` and `Model SHA1` to use your own model */
+    modelName?: 'custom' | 'silero-v5.1.2' | DataLink
+    /* The URL or path of model  
+ We used `ggml` format model, please refer to https://huggingface.co/ggml-org/whisper-vad */
+    modelUrl?: string | DataLink
+    /* Hash type of model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of model */
+    modelHash?: string | DataLink
+    /* Use GPU Acceleration for inference. Currently iOS only. */
+    modelUseGPU?: boolean | DataLink
+    /* Number of threads to use for processing */
+    modelThreads?: number | DataLink
+    /* Speech probability threshold (0.0-1.0) */
+    detectThreshold?: number | DataLink
+    /* Minimum speech duration in milliseconds */
+    detectMinSpeechDurationMs?: number | DataLink
+    /* Minimum silence duration in milliseconds */
+    detectMinSilenceDurationMs?: number | DataLink
+    /* Maximum speech duration in seconds */
+    detectMaxSpeechDurationS?: number | DataLink
+    /* Padding around speech segments in milliseconds */
+    detectSpeechPadMs?: number | DataLink
+    /* Overlap between analysis windows (0.0-1.0) */
+    detectSamplesOverlap?: number | DataLink
+    /* The file URL or path to be analyzed.  
+ It only supported `wav` format with 16kHz sample rate & single (mono) channel */
+    detectFileUrl?: string | DataLink
+    /* MD5 of file to be analyzed */
+    detectFileMd5?: string | DataLink
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+    /* Event triggered when got detection result */
+    onDetected?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Context details */
+    contextDetails?: () => Data
+    /* Is detecting */
+    isDetecting?: () => Data
+    /* Detection segments result */
+    detectionSegments?: () => Data
+    /* Detection details */
+    detectionDetails?: () => Data
+  }
+}
+
+/* Local Voice Activity Detection (VAD) inference based on GGML and [whisper.rn](https://github.com/mybigday/whisper.rn) */
+export type GeneratorVadInference = Generator &
+  GeneratorVadInferenceDef & {
+    templateKey: 'GENERATOR_VAD_INFERENCE'
+    switches: Array<
+      SwitchDef &
+        GeneratorVadInferenceDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'contextDetails'
+                    | 'isDetecting'
+                    | 'detectionSegments'
+                    | 'detectionDetails'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/VectorStore.ts

@@ -0,0 +1,223 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Load the model (Only if source is ggml) */
+export type GeneratorVectorStoreActionModelLoad = Action & {
+  __actionName: 'GENERATOR_VECTOR_STORE_MODEL_LOAD'
+}
+
+/* Release the model (Only if source is ggml) */
+export type GeneratorVectorStoreActionModelRelease = Action & {
+  __actionName: 'GENERATOR_VECTOR_STORE_MODEL_RELEASE'
+}
+
+/* Reset the embedding session */
+export type GeneratorVectorStoreActionReset = ActionWithParams & {
+  __actionName: 'GENERATOR_VECTOR_STORE_RESET'
+  params?: Array<{
+    input: 'resetType'
+    value?: 'session' | 'all' | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Insert file content with path or url, support Office / OpenOffice / PDF  
+
+ PDF is not currently supprted in iOS / Android */
+export type GeneratorVectorStoreActionInsertFile = ActionWithParams & {
+  __actionName: 'GENERATOR_VECTOR_STORE_INSERT_FILE'
+  params?: Array<
+    | {
+        input: 'filePath'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileExtension'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'chunkSize'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Cancel the file download */
+export type GeneratorVectorStoreActionCancelFileDownload = Action & {
+  __actionName: 'GENERATOR_VECTOR_STORE_CANCEL_FILE_DOWNLOAD'
+}
+
+/* Insert pure text content to vector store */
+export type GeneratorVectorStoreActionInsertContent = ActionWithParams & {
+  __actionName: 'GENERATOR_VECTOR_STORE_INSERT_CONTENT'
+  params?: Array<
+    | {
+        input: 'textContent'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'chunkSize'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Remove file from vector store */
+export type GeneratorVectorStoreActionRemoveFile = ActionWithParams & {
+  __actionName: 'GENERATOR_VECTOR_STORE_REMOVE_FILE'
+  params?: Array<{
+    input: 'fileName'
+    value?: string | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Search similar text chunks in vector store */
+export type GeneratorVectorStoreActionSearch = ActionWithParams & {
+  __actionName: 'GENERATOR_VECTOR_STORE_SEARCH'
+  params?: Array<
+    | {
+        input: 'text'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'filename'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'limit'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'threshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'ignoreThreshold'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+interface GeneratorVectorStoreDef {
+  /*
+Default property:
+{
+  "source": "ggml",
+  "ggmlContextSize": 512,
+  "openaiBaseUrl": "https://api.openai.com/v1",
+  "openaiModel": "text-embedding-3-small",
+  "embedSessionId": "default-embeddings",
+  "fileChunkSize": 256
+}
+  */
+  property?: {
+    /* Initialize embedding model context & tokenizer context on subspace mounted */
+    init?: boolean | DataLink
+    /* Source of embedding model. Note that you can't use the saved embedddings data in different sources. */
+    source?: 'ggml' | 'openai' | DataLink
+    /* Path of ggml model */
+    ggmlModelUrl?: string | DataLink
+    /* Hash type of ggml model */
+    ggmlModelHashType?: 'sha256' | 'sha1' | 'md5' | DataLink
+    /* Hash of ggml model */
+    ggmlModelHash?: string | DataLink
+    /* Size of ggml context */
+    ggmlContextSize?: number | DataLink
+    /* Pooling type of ggml model */
+    ggmlPoolingType?: 'none' | 'mean' | 'cls' | 'last' | 'rank' | DataLink
+    /* Accelerator variant of ggml model */
+    ggmlAccelVariant?: 'default' | 'vulkan' | 'cuda' | DataLink
+    /* Normalize */
+    ggmlEmbdNormalize?: number | DataLink
+    /* API key of OpenAI Compatible API */
+    openaiApiKey?: string | DataLink
+    /* Base URL of OpenAI Compatible API. For example, we can use ollama / llama.cpp server instead of openai */
+    openaiBaseUrl?: string | DataLink
+    /* Model of OpenAI Compatible API */
+    openaiModel?: 'text-embedding-3-small' | 'text-embedding-3-large' | DataLink
+    /* Session ID of embedding */
+    embedSessionId?: string | DataLink
+    /* Storage type of embedding */
+    embedStorageType?: 'file' | 'memory' | DataLink
+    /* Tokenizer model URL (Recommended to use same with your LLM model, the model will just load with vocab_only mode) */
+    tokenizerModelUrl?: string | DataLink
+    /* Tokenizer model hash */
+    tokenizerModelHash?: string | DataLink
+    /* Tokenizer model hash type */
+    tokenizerModelHashType?: 'sha256' | 'sha1' | 'md5' | DataLink
+    /* Chunk size to handle file content (token), must be less then embedding model context size */
+    fileChunkSize?: number | DataLink
+  }
+  events?: {
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* undefined */
+    isReady?: () => Data
+    /* Whether the embedding is processing */
+    processing?: () => Data
+    /* Progress of embedding processing */
+    processProgress?: () => Data
+    /* Files inserted to embedding */
+    files?: () => Data
+    /* Search result */
+    searchResult?: () => Data
+  }
+}
+
+/* Vector Store - File embeddings & search */
+export type GeneratorVectorStore = Generator &
+  GeneratorVectorStoreDef & {
+    templateKey: 'GENERATOR_VECTOR_STORE'
+    switches: Array<
+      SwitchDef &
+        GeneratorVectorStoreDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'isReady' | 'processing' | 'processProgress' | 'files' | 'searchResult'
+                  value: any
+                }
+          }>
+        }
+    >
+  }